commit
c44785be57
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -12,6 +12,7 @@
|
|||||||
# Visual Studio
|
# Visual Studio
|
||||||
*.sln text eol=crlf
|
*.sln text eol=crlf
|
||||||
*.vcxproj* text eol=crlf
|
*.vcxproj* text eol=crlf
|
||||||
|
*.vcproj* text eol=crlf
|
||||||
*.suo binary
|
*.suo binary
|
||||||
*.rc binary
|
*.rc binary
|
||||||
|
|
||||||
|
4
.gitignore
vendored
4
.gitignore
vendored
@ -31,7 +31,7 @@ projects/VS2015
|
|||||||
.directory
|
.directory
|
||||||
_codelite/
|
_codelite/
|
||||||
_zstdbench/
|
_zstdbench/
|
||||||
zlib_wrapper/
|
.clang_complete
|
||||||
|
|
||||||
# CMake
|
# CMake
|
||||||
contrib/cmake/
|
projects/cmake/
|
||||||
|
116
.travis.yml
116
.travis.yml
@ -1,34 +1,90 @@
|
|||||||
language: c
|
language: c
|
||||||
|
|
||||||
before_install:
|
|
||||||
- sudo apt-get update -qq
|
|
||||||
- sudo apt-get install -qq clang
|
|
||||||
- sudo apt-get install -qq g++-multilib
|
|
||||||
- sudo apt-get install -qq gcc-multilib
|
|
||||||
- sudo apt-get install -qq valgrind
|
|
||||||
|
|
||||||
env:
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=travis-install
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=cmaketest
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=clangtest
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=gpptest
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=gnu90test
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=c99test
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=gnu99test
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=armtest-w-install
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=test
|
|
||||||
- ZSTD_TRAVIS_CI_ENV="-C programs test32"
|
|
||||||
- ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
|
|
||||||
- ZSTD_TRAVIS_CI_ENV="-C versionsTest"
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=usan
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=asan
|
|
||||||
- ZSTD_TRAVIS_CI_ENV=asan32
|
|
||||||
- ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"
|
|
||||||
|
|
||||||
compiler: gcc
|
compiler: gcc
|
||||||
|
|
||||||
script:
|
|
||||||
- make $ZSTD_TRAVIS_CI_ENV
|
|
||||||
|
|
||||||
matrix:
|
matrix:
|
||||||
fast_finish: true
|
fast_finish: true
|
||||||
|
include:
|
||||||
|
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
|
||||||
|
- os: linux
|
||||||
|
sudo: false
|
||||||
|
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
|
||||||
|
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
|
||||||
|
# - os: linux
|
||||||
|
# sudo: required
|
||||||
|
# env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest"
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
|
||||||
|
- os: linux
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
|
||||||
|
# Ubuntu 14.04 LTS Server Edition 64 bit
|
||||||
|
- os: linux
|
||||||
|
dist: trusty
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
|
||||||
|
- os: linux
|
||||||
|
dist: trusty
|
||||||
|
sudo: required
|
||||||
|
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
|
||||||
|
# OS X Mavericks
|
||||||
|
- os: osx
|
||||||
|
env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
|
||||||
|
- os: osx
|
||||||
|
env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
|
||||||
|
- os: osx
|
||||||
|
env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
|
||||||
|
exclude:
|
||||||
|
- compiler: gcc
|
||||||
|
|
||||||
|
before_install:
|
||||||
|
- set -e
|
||||||
|
- |
|
||||||
|
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
|
||||||
|
CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
|
||||||
|
echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
|
||||||
|
if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
|
||||||
|
sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
script:
|
||||||
|
- make $MAKE_PARAM
|
||||||
|
129
Makefile
129
Makefile
@ -32,6 +32,7 @@
|
|||||||
|
|
||||||
PRGDIR = programs
|
PRGDIR = programs
|
||||||
ZSTDDIR = lib
|
ZSTDDIR = lib
|
||||||
|
ZWRAPDIR = zlibWrapper
|
||||||
|
|
||||||
# Define nul output
|
# Define nul output
|
||||||
ifneq (,$(filter Windows%,$(OS)))
|
ifneq (,$(filter Windows%,$(OS)))
|
||||||
@ -40,7 +41,7 @@ else
|
|||||||
VOID = /dev/null
|
VOID = /dev/null
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
|
.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
|
||||||
|
|
||||||
default: zstdprogram
|
default: zstdprogram
|
||||||
|
|
||||||
@ -51,19 +52,25 @@ all:
|
|||||||
zstdprogram:
|
zstdprogram:
|
||||||
$(MAKE) -C $(PRGDIR)
|
$(MAKE) -C $(PRGDIR)
|
||||||
|
|
||||||
|
zlibwrapper:
|
||||||
|
$(MAKE) -C $(ZSTDDIR) all
|
||||||
|
$(MAKE) -C $(ZWRAPDIR) all
|
||||||
|
|
||||||
test:
|
test:
|
||||||
$(MAKE) -C $(PRGDIR) $@
|
$(MAKE) -C $(PRGDIR) $@
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||||
|
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||||
@echo Cleaning completed
|
@echo Cleaning completed
|
||||||
|
|
||||||
|
|
||||||
#------------------------------------------------------------------------
|
#------------------------------------------------------------------------
|
||||||
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
||||||
|
#------------------------------------------------------------------------
|
||||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
||||||
|
HOST_OS = POSIX
|
||||||
install:
|
install:
|
||||||
$(MAKE) -C $(ZSTDDIR) $@
|
$(MAKE) -C $(ZSTDDIR) $@
|
||||||
$(MAKE) -C $(PRGDIR) $@
|
$(MAKE) -C $(PRGDIR) $@
|
||||||
@ -75,15 +82,80 @@ uninstall:
|
|||||||
travis-install:
|
travis-install:
|
||||||
$(MAKE) install PREFIX=~/install_test_dir
|
$(MAKE) install PREFIX=~/install_test_dir
|
||||||
|
|
||||||
cmaketest:
|
gpptest: clean
|
||||||
cd projects/cmake/build ; cmake .. ; $(MAKE)
|
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||||
|
|
||||||
clangtest: clean
|
clangtest: clean
|
||||||
clang -v
|
clang -v
|
||||||
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
|
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
|
||||||
|
|
||||||
gpptest: clean
|
armtest: clean
|
||||||
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||||
|
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||||
|
|
||||||
|
# for Travis CI
|
||||||
|
arminstall: clean
|
||||||
|
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
|
||||||
|
|
||||||
|
# for Travis CI
|
||||||
|
armtest-w-install: clean arminstall armtest
|
||||||
|
|
||||||
|
ppctest: clean
|
||||||
|
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||||
|
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||||
|
|
||||||
|
# for Travis CI
|
||||||
|
ppcinstall: clean
|
||||||
|
# sudo apt-get update -y -q
|
||||||
|
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
|
||||||
|
|
||||||
|
# for Travis CI
|
||||||
|
ppctest-w-install: clean ppcinstall ppctest
|
||||||
|
|
||||||
|
ppc64test: clean
|
||||||
|
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||||
|
$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||||
|
|
||||||
|
ppc64install: clean
|
||||||
|
sudo apt-get update -y -q
|
||||||
|
sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
|
||||||
|
update-binfmts --displ
|
||||||
|
|
||||||
|
ppc64test-w-install: clean ppc64install ppc64test
|
||||||
|
|
||||||
|
usan: clean
|
||||||
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
|
||||||
|
|
||||||
|
asan: clean
|
||||||
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||||
|
|
||||||
|
msan: clean
|
||||||
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
|
||||||
|
|
||||||
|
asan32: clean
|
||||||
|
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||||
|
|
||||||
|
uasan: clean
|
||||||
|
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
|
||||||
|
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||||
|
HOST_OS = MSYS
|
||||||
|
CMAKE_PARAMS = -G"MSYS Makefiles"
|
||||||
|
endif
|
||||||
|
|
||||||
|
|
||||||
|
#------------------------------------------------------------------------
|
||||||
|
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
|
||||||
|
#------------------------------------------------------------------------
|
||||||
|
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||||
|
cmaketest:
|
||||||
|
cmake --version
|
||||||
|
rm -rf projects/cmake/build
|
||||||
|
mkdir projects/cmake/build
|
||||||
|
cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||||
|
|
||||||
c90test: clean
|
c90test: clean
|
||||||
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
||||||
@ -108,49 +180,4 @@ bmix32test: clean
|
|||||||
|
|
||||||
bmi32test: clean
|
bmi32test: clean
|
||||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
|
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
|
||||||
|
|
||||||
armtest: clean
|
|
||||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
|
||||||
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
|
||||||
|
|
||||||
# for Travis CI
|
|
||||||
arminstall: clean
|
|
||||||
sudo apt-get install -q qemu
|
|
||||||
sudo apt-get install -q binfmt-support
|
|
||||||
sudo apt-get install -q qemu-user-static
|
|
||||||
sudo apt-get install -q gcc-arm-linux-gnueabi
|
|
||||||
|
|
||||||
# for Travis CI
|
|
||||||
armtest-w-install: clean arminstall armtest
|
|
||||||
|
|
||||||
ppctest: clean
|
|
||||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
|
||||||
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
|
||||||
|
|
||||||
# for Travis CI
|
|
||||||
ppcinstall: clean
|
|
||||||
sudo apt-get install -q qemu
|
|
||||||
sudo apt-get install -q binfmt-support
|
|
||||||
sudo apt-get install -q qemu-user-static
|
|
||||||
sudo apt-get update -q
|
|
||||||
sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available)
|
|
||||||
|
|
||||||
# for Travis CI
|
|
||||||
ppctest-w-install: clean ppcinstall ppctest
|
|
||||||
|
|
||||||
usan: clean
|
|
||||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
|
|
||||||
|
|
||||||
asan: clean
|
|
||||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
|
||||||
|
|
||||||
msan: clean
|
|
||||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
|
|
||||||
|
|
||||||
asan32: clean
|
|
||||||
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
|
||||||
|
|
||||||
uasan: clean
|
|
||||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
|
|
||||||
|
|
||||||
endif
|
endif
|
||||||
|
10
NEWS
10
NEWS
@ -1,8 +1,12 @@
|
|||||||
v0.6.2
|
v0.7.0
|
||||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||||
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
||||||
New : Support pass-through mode, when using `-df`
|
New : Frame checksum support
|
||||||
New : Support for custom malloc/free functions
|
New : Support pass-through mode (when using `-df`)
|
||||||
|
New : API : dictionary files from custom content, by Giuseppe Ottaviano
|
||||||
|
New : API support for custom malloc/free functions
|
||||||
|
New : controllable Dictionary ID
|
||||||
|
New : Support for skippable frames
|
||||||
|
|
||||||
v0.6.1
|
v0.6.1
|
||||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||||
|
35
appveyor.yml
35
appveyor.yml
@ -1,24 +1,24 @@
|
|||||||
version: 1.0.{build}
|
version: 1.0.{build}
|
||||||
environment:
|
environment:
|
||||||
matrix:
|
matrix:
|
||||||
|
- COMPILER: "visual"
|
||||||
|
CONFIGURATION: "Debug"
|
||||||
|
PLATFORM: "x64"
|
||||||
|
- COMPILER: "visual"
|
||||||
|
CONFIGURATION: "Debug"
|
||||||
|
PLATFORM: "Win32"
|
||||||
|
- COMPILER: "visual"
|
||||||
|
CONFIGURATION: "Release"
|
||||||
|
PLATFORM: "x64"
|
||||||
|
- COMPILER: "visual"
|
||||||
|
CONFIGURATION: "Release"
|
||||||
|
PLATFORM: "Win32"
|
||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
MAKE_PARAMS: "test"
|
MAKE_PARAMS: "test"
|
||||||
PLATFORM: "mingw64"
|
PLATFORM: "mingw64"
|
||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
MAKE_PARAMS: "test"
|
MAKE_PARAMS: "test"
|
||||||
PLATFORM: "mingw32"
|
PLATFORM: "mingw32"
|
||||||
- COMPILER: "visual"
|
|
||||||
CONFIGURATION: "Release"
|
|
||||||
PLATFORM: "Win32"
|
|
||||||
- COMPILER: "visual"
|
|
||||||
CONFIGURATION: "Release"
|
|
||||||
PLATFORM: "x64"
|
|
||||||
- COMPILER: "visual"
|
|
||||||
CONFIGURATION: "Debug"
|
|
||||||
PLATFORM: "Win32"
|
|
||||||
- COMPILER: "visual"
|
|
||||||
CONFIGURATION: "Debug"
|
|
||||||
PLATFORM: "x64"
|
|
||||||
|
|
||||||
install:
|
install:
|
||||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||||
@ -80,7 +80,7 @@ build_script:
|
|||||||
ECHO *** &&
|
ECHO *** &&
|
||||||
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\
|
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
|
||||||
)
|
)
|
||||||
|
|
||||||
test_script:
|
test_script:
|
||||||
@ -91,12 +91,11 @@ test_script:
|
|||||||
)
|
)
|
||||||
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
|
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||||
CD programs &&
|
CD programs &&
|
||||||
SET ZSTD=..\projects\zstd.exe &&
|
SET ZSTD=./zstd.exe &&
|
||||||
SET DATAGEN=..\projects\datagen.exe &&
|
sh -e playTests.sh --test-large-data &&
|
||||||
CALL tests\playTests.bat --test-large-data &&
|
fullbench.exe -i1 &&
|
||||||
|
fullbench.exe -i1 -P0 &&
|
||||||
CD .. &&
|
CD .. &&
|
||||||
projects\fullbench.exe -i1 &&
|
|
||||||
projects\fullbench.exe -i1 -P0 &&
|
|
||||||
projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||||
projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||||
projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||||
|
19
lib/Makefile
19
lib/Makefile
@ -43,24 +43,21 @@ VERSION?= $(LIBVER)
|
|||||||
|
|
||||||
DESTDIR?=
|
DESTDIR?=
|
||||||
PREFIX ?= /usr/local
|
PREFIX ?= /usr/local
|
||||||
CPPFLAGS= -I./common
|
|
||||||
CFLAGS ?= -O3
|
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
|
|
||||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
|
||||||
|
|
||||||
LIBDIR ?= $(PREFIX)/lib
|
LIBDIR ?= $(PREFIX)/lib
|
||||||
INCLUDEDIR=$(PREFIX)/include
|
INCLUDEDIR=$(PREFIX)/include
|
||||||
|
|
||||||
ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c
|
CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
|
||||||
ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c
|
CFLAGS ?= -O3
|
||||||
ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES)
|
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||||
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
|
|
||||||
|
|
||||||
|
ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
|
||||||
|
|
||||||
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||||
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
||||||
else
|
else
|
||||||
ZSTD_FILES+= $(ZSTD_LEGACY)
|
ZSTD_FILES+= legacy/*.c
|
||||||
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
|
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
@ -12,13 +12,10 @@ To build the zstd library the following files are required:
|
|||||||
- [common/error_public.h](common/error_public.h)
|
- [common/error_public.h](common/error_public.h)
|
||||||
- common/fse.h
|
- common/fse.h
|
||||||
- common/fse_decompress.c
|
- common/fse_decompress.c
|
||||||
- common/fse_static.h
|
|
||||||
- common/huf.h
|
- common/huf.h
|
||||||
- common/huf_static.h
|
|
||||||
- [common/mem.h](common/mem.h)
|
- [common/mem.h](common/mem.h)
|
||||||
- [common/zstd.h]
|
- [common/zstd.h]
|
||||||
- common/zstd_internal.h
|
- common/zstd_internal.h
|
||||||
- common/zstd_static.h
|
|
||||||
- compress/fse_compress.c
|
- compress/fse_compress.c
|
||||||
- compress/huf_compress.c
|
- compress/huf_compress.c
|
||||||
- compress/zstd_compress.c
|
- compress/zstd_compress.c
|
||||||
@ -27,12 +24,10 @@ To build the zstd library the following files are required:
|
|||||||
- decompress/zstd_decompress.c
|
- decompress/zstd_decompress.c
|
||||||
|
|
||||||
Stable API is exposed in [common/zstd.h].
|
Stable API is exposed in [common/zstd.h].
|
||||||
Advanced and experimental API is exposed in [common/zstd_static.h].
|
Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`.
|
||||||
API elements of [common/zstd_static.h] should be used with static linking only,
|
Never use them with a dynamic library, as their definition may change in future versions.
|
||||||
as their definition may change in future version of the library.
|
|
||||||
|
|
||||||
[common/zstd.h]: common/zstd.h
|
[common/zstd.h]: common/zstd.h
|
||||||
[common/zstd_static.h]: common/zstd_static.h
|
|
||||||
|
|
||||||
|
|
||||||
#### Separate compressor and decompressor
|
#### Separate compressor and decompressor
|
||||||
@ -47,7 +42,6 @@ This complementary API makes streaming integration easier.
|
|||||||
It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
|
It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
|
||||||
|
|
||||||
- common/zbuff.h
|
- common/zbuff.h
|
||||||
- common/zbuff_static.h
|
|
||||||
- compress/zbuff_compress.c
|
- compress/zbuff_compress.c
|
||||||
- decompress/zbuff_decompress.c
|
- decompress/zbuff_decompress.c
|
||||||
|
|
||||||
@ -59,7 +53,6 @@ To create dictionaries from training sets :
|
|||||||
- dictBuilder/divsufsort.h
|
- dictBuilder/divsufsort.h
|
||||||
- dictBuilder/zdict.c
|
- dictBuilder/zdict.c
|
||||||
- dictBuilder/zdict.h
|
- dictBuilder/zdict.h
|
||||||
- dictBuilder/zdict_static.h
|
|
||||||
|
|
||||||
#### Miscellaneous
|
#### Miscellaneous
|
||||||
|
|
||||||
@ -68,4 +61,3 @@ The other files are not source code. There are :
|
|||||||
- LICENSE : contains the BSD license text
|
- LICENSE : contains the BSD license text
|
||||||
- Makefile : script to compile or install zstd library (static or dynamic)
|
- Makefile : script to compile or install zstd library (static or dynamic)
|
||||||
- libzstd.pc.in : for pkg-config (make install)
|
- libzstd.pc.in : for pkg-config (make install)
|
||||||
|
|
||||||
|
@ -35,12 +35,12 @@
|
|||||||
/* *************************************
|
/* *************************************
|
||||||
* Dependencies
|
* Dependencies
|
||||||
***************************************/
|
***************************************/
|
||||||
#include <stdlib.h>
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "fse_static.h" /* FSE_MIN_TABLELOG */
|
#include "error_private.h" /* ERR_*, ERROR */
|
||||||
#include "error_private.h"
|
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
|
||||||
#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */
|
#include "fse.h" /* FSE_isError, FSE_getErrorName */
|
||||||
#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */
|
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
|
||||||
|
#include "huf.h" /* HUF_isError, HUF_getErrorName */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -156,3 +156,76 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
|||||||
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
||||||
return ip-istart;
|
return ip-istart;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*! HUF_readStats() :
|
||||||
|
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||||
|
`huffWeight` is destination buffer.
|
||||||
|
@return : size read from `src` , or an error Code .
|
||||||
|
Note : Needed by HUF_readCTable() and HUF_readDTableXn() .
|
||||||
|
*/
|
||||||
|
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||||
|
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||||
|
const void* src, size_t srcSize)
|
||||||
|
{
|
||||||
|
U32 weightTotal;
|
||||||
|
const BYTE* ip = (const BYTE*) src;
|
||||||
|
size_t iSize = ip[0];
|
||||||
|
size_t oSize;
|
||||||
|
|
||||||
|
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||||
|
|
||||||
|
if (iSize >= 128) { /* special header */
|
||||||
|
if (iSize >= (242)) { /* RLE */
|
||||||
|
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||||
|
oSize = l[iSize-242];
|
||||||
|
memset(huffWeight, 1, hwSize);
|
||||||
|
iSize = 0;
|
||||||
|
}
|
||||||
|
else { /* Incompressible */
|
||||||
|
oSize = iSize - 127;
|
||||||
|
iSize = ((oSize+1)/2);
|
||||||
|
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||||
|
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||||
|
ip += 1;
|
||||||
|
{ U32 n;
|
||||||
|
for (n=0; n<oSize; n+=2) {
|
||||||
|
huffWeight[n] = ip[n/2] >> 4;
|
||||||
|
huffWeight[n+1] = ip[n/2] & 15;
|
||||||
|
} } } }
|
||||||
|
else { /* header compressed with FSE (normal case) */
|
||||||
|
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||||
|
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||||
|
if (FSE_isError(oSize)) return oSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* collect weight stats */
|
||||||
|
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
|
||||||
|
weightTotal = 0;
|
||||||
|
{ U32 n; for (n=0; n<oSize; n++) {
|
||||||
|
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||||
|
rankStats[huffWeight[n]]++;
|
||||||
|
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||||
|
} }
|
||||||
|
|
||||||
|
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||||
|
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||||
|
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||||
|
*tableLogPtr = tableLog;
|
||||||
|
/* determine last weight */
|
||||||
|
{ U32 const total = 1 << tableLog;
|
||||||
|
U32 const rest = total - weightTotal;
|
||||||
|
U32 const verif = 1 << BIT_highbit32(rest);
|
||||||
|
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||||
|
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||||
|
huffWeight[oSize] = (BYTE)lastWeight;
|
||||||
|
rankStats[lastWeight]++;
|
||||||
|
} }
|
||||||
|
|
||||||
|
/* check tree construction validity */
|
||||||
|
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||||
|
|
||||||
|
/* results */
|
||||||
|
*nbSymbolsPtr = (U32)(oSize+1);
|
||||||
|
return iSize+1;
|
||||||
|
}
|
||||||
|
@ -102,10 +102,12 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
|||||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||||
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
||||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||||
|
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||||
|
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||||
case PREFIX(maxCode):
|
case PREFIX(maxCode):
|
||||||
default: return notErrorCode;
|
default: return notErrorCode;
|
||||||
}
|
}
|
||||||
|
@ -54,10 +54,12 @@ typedef enum {
|
|||||||
ZSTD_error_dstSize_tooSmall,
|
ZSTD_error_dstSize_tooSmall,
|
||||||
ZSTD_error_srcSize_wrong,
|
ZSTD_error_srcSize_wrong,
|
||||||
ZSTD_error_corruption_detected,
|
ZSTD_error_corruption_detected,
|
||||||
|
ZSTD_error_checksum_wrong,
|
||||||
ZSTD_error_tableLog_tooLarge,
|
ZSTD_error_tableLog_tooLarge,
|
||||||
ZSTD_error_maxSymbolValue_tooLarge,
|
ZSTD_error_maxSymbolValue_tooLarge,
|
||||||
ZSTD_error_maxSymbolValue_tooSmall,
|
ZSTD_error_maxSymbolValue_tooSmall,
|
||||||
ZSTD_error_dictionary_corrupted,
|
ZSTD_error_dictionary_corrupted,
|
||||||
|
ZSTD_error_dictionary_wrong,
|
||||||
ZSTD_error_maxCode
|
ZSTD_error_maxCode
|
||||||
} ZSTD_ErrorCode;
|
} ZSTD_ErrorCode;
|
||||||
|
|
||||||
|
349
lib/common/fse.h
349
lib/common/fse.h
@ -272,6 +272,355 @@ If there is an error, the function will return an error code, which can be teste
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef FSE_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
/* *** Dependency *** */
|
||||||
|
#include "bitstream.h"
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* Static allocation
|
||||||
|
*******************************************/
|
||||||
|
/* FSE buffer bounds */
|
||||||
|
#define FSE_NCOUNTBOUND 512
|
||||||
|
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
||||||
|
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||||
|
|
||||||
|
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
||||||
|
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||||
|
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* FSE advanced API
|
||||||
|
*******************************************/
|
||||||
|
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||||
|
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||||
|
|
||||||
|
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||||
|
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
||||||
|
|
||||||
|
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||||
|
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
|
||||||
|
|
||||||
|
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||||
|
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
||||||
|
|
||||||
|
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||||
|
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||||
|
|
||||||
|
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||||
|
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* FSE symbol compression API
|
||||||
|
*******************************************/
|
||||||
|
/*!
|
||||||
|
This API consists of small unitary functions, which highly benefit from being inlined.
|
||||||
|
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
||||||
|
Visual seems to do it automatically.
|
||||||
|
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
||||||
|
If none of these solutions is applicable, include "fse.c" directly.
|
||||||
|
*/
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
ptrdiff_t value;
|
||||||
|
const void* stateTable;
|
||||||
|
const void* symbolTT;
|
||||||
|
unsigned stateLog;
|
||||||
|
} FSE_CState_t;
|
||||||
|
|
||||||
|
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||||
|
|
||||||
|
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||||
|
|
||||||
|
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||||
|
|
||||||
|
/**<
|
||||||
|
These functions are inner components of FSE_compress_usingCTable().
|
||||||
|
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
||||||
|
|
||||||
|
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
||||||
|
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
||||||
|
|
||||||
|
You will need a few variables to track your CStream. They are :
|
||||||
|
|
||||||
|
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||||
|
BIT_CStream_t bitStream; // bitStream tracking structure
|
||||||
|
FSE_CState_t state; // State tracking structure (can have several)
|
||||||
|
|
||||||
|
|
||||||
|
The first thing to do is to init bitStream and state.
|
||||||
|
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||||
|
FSE_initCState(&state, ct);
|
||||||
|
|
||||||
|
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||||
|
You can then encode your input data, byte after byte.
|
||||||
|
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||||
|
Remember decoding will be done in reverse direction.
|
||||||
|
FSE_encodeByte(&bitStream, &state, symbol);
|
||||||
|
|
||||||
|
At any time, you can also add any bit sequence.
|
||||||
|
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
||||||
|
BIT_addBits(&bitStream, bitField, nbBits);
|
||||||
|
|
||||||
|
The above methods don't commit data to memory, they just store it into local register, for speed.
|
||||||
|
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||||
|
Writing data to memory is a manual operation, performed by the flushBits function.
|
||||||
|
BIT_flushBits(&bitStream);
|
||||||
|
|
||||||
|
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||||
|
FSE_flushState(&bitStream, &state);
|
||||||
|
|
||||||
|
Finally, you must close the bitStream.
|
||||||
|
The function returns the size of CStream in bytes.
|
||||||
|
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||||
|
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||||
|
size_t size = BIT_closeCStream(&bitStream);
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* FSE symbol decompression API
|
||||||
|
*******************************************/
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
size_t state;
|
||||||
|
const void* table; /* precise table may vary, depending on U16 */
|
||||||
|
} FSE_DState_t;
|
||||||
|
|
||||||
|
|
||||||
|
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
||||||
|
|
||||||
|
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||||
|
|
||||||
|
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||||
|
|
||||||
|
/**<
|
||||||
|
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||||
|
You will decode FSE-encoded symbols from the bitStream,
|
||||||
|
and also any other bitFields you put in, **in reverse order**.
|
||||||
|
|
||||||
|
You will need a few variables to track your bitStream. They are :
|
||||||
|
|
||||||
|
BIT_DStream_t DStream; // Stream context
|
||||||
|
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||||
|
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||||
|
|
||||||
|
The first thing to do is to init the bitStream.
|
||||||
|
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||||
|
|
||||||
|
You should then retrieve your initial state(s)
|
||||||
|
(in reverse flushing order if you have several ones) :
|
||||||
|
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||||
|
|
||||||
|
You can then decode your data, symbol after symbol.
|
||||||
|
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||||
|
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||||
|
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||||
|
|
||||||
|
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||||
|
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||||
|
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||||
|
|
||||||
|
All above operations only read from local register (which size depends on size_t).
|
||||||
|
Refueling the register from memory is manually performed by the reload method.
|
||||||
|
endSignal = FSE_reloadDStream(&DStream);
|
||||||
|
|
||||||
|
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||||
|
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||||
|
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||||
|
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||||
|
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||||
|
|
||||||
|
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||||
|
to properly detect the exact end of stream.
|
||||||
|
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||||
|
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||||
|
|
||||||
|
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||||
|
Checking if DStream has reached its end is performed by :
|
||||||
|
BIT_endOfDStream(&DStream);
|
||||||
|
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||||
|
FSE_endOfDState(&DState);
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* FSE unsafe API
|
||||||
|
*******************************************/
|
||||||
|
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||||
|
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||||
|
|
||||||
|
|
||||||
|
/* *****************************************
|
||||||
|
* Implementation of inlined functions
|
||||||
|
*******************************************/
|
||||||
|
typedef struct {
|
||||||
|
int deltaFindState;
|
||||||
|
U32 deltaNbBits;
|
||||||
|
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
||||||
|
|
||||||
|
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||||
|
{
|
||||||
|
const void* ptr = ct;
|
||||||
|
const U16* u16ptr = (const U16*) ptr;
|
||||||
|
const U32 tableLog = MEM_read16(ptr);
|
||||||
|
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||||
|
statePtr->stateTable = u16ptr+2;
|
||||||
|
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||||
|
statePtr->stateLog = tableLog;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*! FSE_initCState2() :
|
||||||
|
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
|
||||||
|
* uses the smallest state value possible, saving the cost of this symbol */
|
||||||
|
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
|
||||||
|
{
|
||||||
|
FSE_initCState(statePtr, ct);
|
||||||
|
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||||
|
const U16* stateTable = (const U16*)(statePtr->stateTable);
|
||||||
|
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
|
||||||
|
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
|
||||||
|
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||||
|
{
|
||||||
|
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||||
|
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||||
|
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||||
|
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||||
|
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||||
|
{
|
||||||
|
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||||
|
BIT_flushBits(bitC);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*<===== Decompression =====>*/
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U16 tableLog;
|
||||||
|
U16 fastMode;
|
||||||
|
} FSE_DTableHeader; /* sizeof U32 */
|
||||||
|
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
unsigned short newState;
|
||||||
|
unsigned char symbol;
|
||||||
|
unsigned char nbBits;
|
||||||
|
} FSE_decode_t; /* size == U32 */
|
||||||
|
|
||||||
|
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
||||||
|
{
|
||||||
|
const void* ptr = dt;
|
||||||
|
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
|
||||||
|
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||||
|
BIT_reloadDStream(bitD);
|
||||||
|
DStatePtr->table = dt + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
||||||
|
{
|
||||||
|
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||||
|
return DInfo.symbol;
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||||
|
{
|
||||||
|
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||||
|
U32 const nbBits = DInfo.nbBits;
|
||||||
|
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||||
|
DStatePtr->state = DInfo.newState + lowBits;
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||||
|
{
|
||||||
|
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||||
|
U32 const nbBits = DInfo.nbBits;
|
||||||
|
BYTE const symbol = DInfo.symbol;
|
||||||
|
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||||
|
|
||||||
|
DStatePtr->state = DInfo.newState + lowBits;
|
||||||
|
return symbol;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*! FSE_decodeSymbolFast() :
|
||||||
|
unsafe, only works if no symbol has a probability > 50% */
|
||||||
|
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||||
|
{
|
||||||
|
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||||
|
U32 const nbBits = DInfo.nbBits;
|
||||||
|
BYTE const symbol = DInfo.symbol;
|
||||||
|
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||||
|
|
||||||
|
DStatePtr->state = DInfo.newState + lowBits;
|
||||||
|
return symbol;
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||||
|
{
|
||||||
|
return DStatePtr->state == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef FSE_COMMONDEFS_ONLY
|
||||||
|
|
||||||
|
/* **************************************************************
|
||||||
|
* Tuning parameters
|
||||||
|
****************************************************************/
|
||||||
|
/*!MEMORY_USAGE :
|
||||||
|
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||||
|
* Increasing memory usage improves compression ratio
|
||||||
|
* Reduced memory usage can improve speed, due to cache effect
|
||||||
|
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||||
|
#define FSE_MAX_MEMORY_USAGE 14
|
||||||
|
#define FSE_DEFAULT_MEMORY_USAGE 13
|
||||||
|
|
||||||
|
/*!FSE_MAX_SYMBOL_VALUE :
|
||||||
|
* Maximum symbol value authorized.
|
||||||
|
* Required for proper stack allocation */
|
||||||
|
#define FSE_MAX_SYMBOL_VALUE 255
|
||||||
|
|
||||||
|
|
||||||
|
/* **************************************************************
|
||||||
|
* template functions type & suffix
|
||||||
|
****************************************************************/
|
||||||
|
#define FSE_FUNCTION_TYPE BYTE
|
||||||
|
#define FSE_FUNCTION_EXTENSION
|
||||||
|
#define FSE_DECODE_TYPE FSE_decode_t
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||||
|
|
||||||
|
|
||||||
|
/* ***************************************************************
|
||||||
|
* Constants
|
||||||
|
*****************************************************************/
|
||||||
|
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||||
|
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||||
|
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||||
|
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||||
|
#define FSE_MIN_TABLELOG 5
|
||||||
|
|
||||||
|
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||||
|
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||||
|
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* FSE_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -58,7 +58,8 @@
|
|||||||
#include <string.h> /* memcpy, memset */
|
#include <string.h> /* memcpy, memset */
|
||||||
#include <stdio.h> /* printf (debug) */
|
#include <stdio.h> /* printf (debug) */
|
||||||
#include "bitstream.h"
|
#include "bitstream.h"
|
||||||
#include "fse_static.h"
|
#define FSE_STATIC_LINKING_ONLY
|
||||||
|
#include "fse.h"
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
/* **************************************************************
|
||||||
|
@ -1,392 +0,0 @@
|
|||||||
/* ******************************************************************
|
|
||||||
FSE : Finite State Entropy coder
|
|
||||||
header file for static linking (only)
|
|
||||||
Copyright (C) 2013-2015, Yann Collet
|
|
||||||
|
|
||||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
* Redistributions in binary form must reproduce the above
|
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
|
||||||
in the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
You can contact the author at :
|
|
||||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
|
||||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
|
||||||
****************************************************************** */
|
|
||||||
#ifndef FSE_STATIC_H
|
|
||||||
#define FSE_STATIC_H
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* Dependencies
|
|
||||||
*******************************************/
|
|
||||||
#include "fse.h"
|
|
||||||
#include "bitstream.h"
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* Static allocation
|
|
||||||
*******************************************/
|
|
||||||
/* FSE buffer bounds */
|
|
||||||
#define FSE_NCOUNTBOUND 512
|
|
||||||
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
|
||||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
|
||||||
|
|
||||||
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
|
||||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
|
||||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* FSE advanced API
|
|
||||||
*******************************************/
|
|
||||||
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
|
||||||
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
|
||||||
|
|
||||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
|
||||||
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
|
||||||
|
|
||||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
|
||||||
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
|
|
||||||
|
|
||||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
|
||||||
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
|
||||||
|
|
||||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
|
||||||
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
|
||||||
|
|
||||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
|
||||||
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* FSE symbol compression API
|
|
||||||
*******************************************/
|
|
||||||
/*!
|
|
||||||
This API consists of small unitary functions, which highly benefit from being inlined.
|
|
||||||
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
|
||||||
Visual seems to do it automatically.
|
|
||||||
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
|
||||||
If none of these solutions is applicable, include "fse.c" directly.
|
|
||||||
*/
|
|
||||||
typedef struct
|
|
||||||
{
|
|
||||||
ptrdiff_t value;
|
|
||||||
const void* stateTable;
|
|
||||||
const void* symbolTT;
|
|
||||||
unsigned stateLog;
|
|
||||||
} FSE_CState_t;
|
|
||||||
|
|
||||||
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
|
||||||
|
|
||||||
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
|
||||||
|
|
||||||
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
|
||||||
|
|
||||||
/**<
|
|
||||||
These functions are inner components of FSE_compress_usingCTable().
|
|
||||||
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
|
||||||
|
|
||||||
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
|
||||||
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
|
||||||
|
|
||||||
You will need a few variables to track your CStream. They are :
|
|
||||||
|
|
||||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
|
||||||
BIT_CStream_t bitStream; // bitStream tracking structure
|
|
||||||
FSE_CState_t state; // State tracking structure (can have several)
|
|
||||||
|
|
||||||
|
|
||||||
The first thing to do is to init bitStream and state.
|
|
||||||
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
|
||||||
FSE_initCState(&state, ct);
|
|
||||||
|
|
||||||
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
|
||||||
You can then encode your input data, byte after byte.
|
|
||||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
|
||||||
Remember decoding will be done in reverse direction.
|
|
||||||
FSE_encodeByte(&bitStream, &state, symbol);
|
|
||||||
|
|
||||||
At any time, you can also add any bit sequence.
|
|
||||||
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
|
||||||
BIT_addBits(&bitStream, bitField, nbBits);
|
|
||||||
|
|
||||||
The above methods don't commit data to memory, they just store it into local register, for speed.
|
|
||||||
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
|
||||||
Writing data to memory is a manual operation, performed by the flushBits function.
|
|
||||||
BIT_flushBits(&bitStream);
|
|
||||||
|
|
||||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
|
||||||
FSE_flushState(&bitStream, &state);
|
|
||||||
|
|
||||||
Finally, you must close the bitStream.
|
|
||||||
The function returns the size of CStream in bytes.
|
|
||||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
|
||||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
|
||||||
size_t size = BIT_closeCStream(&bitStream);
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* FSE symbol decompression API
|
|
||||||
*******************************************/
|
|
||||||
typedef struct
|
|
||||||
{
|
|
||||||
size_t state;
|
|
||||||
const void* table; /* precise table may vary, depending on U16 */
|
|
||||||
} FSE_DState_t;
|
|
||||||
|
|
||||||
|
|
||||||
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
|
||||||
|
|
||||||
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
|
||||||
|
|
||||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
|
||||||
|
|
||||||
/**<
|
|
||||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
|
||||||
You will decode FSE-encoded symbols from the bitStream,
|
|
||||||
and also any other bitFields you put in, **in reverse order**.
|
|
||||||
|
|
||||||
You will need a few variables to track your bitStream. They are :
|
|
||||||
|
|
||||||
BIT_DStream_t DStream; // Stream context
|
|
||||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
|
||||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
|
||||||
|
|
||||||
The first thing to do is to init the bitStream.
|
|
||||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
|
||||||
|
|
||||||
You should then retrieve your initial state(s)
|
|
||||||
(in reverse flushing order if you have several ones) :
|
|
||||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
|
||||||
|
|
||||||
You can then decode your data, symbol after symbol.
|
|
||||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
|
||||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
|
||||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
|
||||||
|
|
||||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
|
||||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
|
||||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
|
||||||
|
|
||||||
All above operations only read from local register (which size depends on size_t).
|
|
||||||
Refueling the register from memory is manually performed by the reload method.
|
|
||||||
endSignal = FSE_reloadDStream(&DStream);
|
|
||||||
|
|
||||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
|
||||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
|
||||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
|
||||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
|
||||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
|
||||||
|
|
||||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
|
||||||
to properly detect the exact end of stream.
|
|
||||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
|
||||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
|
||||||
|
|
||||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
|
||||||
Checking if DStream has reached its end is performed by :
|
|
||||||
BIT_endOfDStream(&DStream);
|
|
||||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
|
||||||
FSE_endOfDState(&DState);
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* FSE unsafe API
|
|
||||||
*******************************************/
|
|
||||||
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
|
||||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
|
||||||
|
|
||||||
|
|
||||||
/* *****************************************
|
|
||||||
* Implementation of inlined functions
|
|
||||||
*******************************************/
|
|
||||||
typedef struct {
|
|
||||||
int deltaFindState;
|
|
||||||
U32 deltaNbBits;
|
|
||||||
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
|
||||||
{
|
|
||||||
const void* ptr = ct;
|
|
||||||
const U16* u16ptr = (const U16*) ptr;
|
|
||||||
const U32 tableLog = MEM_read16(ptr);
|
|
||||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
|
||||||
statePtr->stateTable = u16ptr+2;
|
|
||||||
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
|
||||||
statePtr->stateLog = tableLog;
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
|
|
||||||
{
|
|
||||||
FSE_initCState(statePtr, ct);
|
|
||||||
{
|
|
||||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
|
||||||
const U16* stateTable = (const U16*)(statePtr->stateTable);
|
|
||||||
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
|
|
||||||
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
|
|
||||||
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
|
||||||
{
|
|
||||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
|
||||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
|
||||||
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
|
||||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
|
||||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
|
||||||
{
|
|
||||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
|
||||||
BIT_flushBits(bitC);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*<===== Decompression =====>*/
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
U16 tableLog;
|
|
||||||
U16 fastMode;
|
|
||||||
} FSE_DTableHeader; /* sizeof U32 */
|
|
||||||
|
|
||||||
typedef struct
|
|
||||||
{
|
|
||||||
unsigned short newState;
|
|
||||||
unsigned char symbol;
|
|
||||||
unsigned char nbBits;
|
|
||||||
} FSE_decode_t; /* size == U32 */
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
|
||||||
{
|
|
||||||
const void* ptr = dt;
|
|
||||||
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
|
|
||||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
|
||||||
BIT_reloadDStream(bitD);
|
|
||||||
DStatePtr->table = dt + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
|
||||||
{
|
|
||||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
|
||||||
return DInfo.symbol;
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
|
||||||
{
|
|
||||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
|
||||||
U32 const nbBits = DInfo.nbBits;
|
|
||||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
|
||||||
DStatePtr->state = DInfo.newState + lowBits;
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
|
||||||
{
|
|
||||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
|
||||||
U32 const nbBits = DInfo.nbBits;
|
|
||||||
BYTE const symbol = DInfo.symbol;
|
|
||||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
|
||||||
|
|
||||||
DStatePtr->state = DInfo.newState + lowBits;
|
|
||||||
return symbol;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*! FSE_decodeSymbolFast() :
|
|
||||||
unsafe, only works if no symbol has a probability > 50% */
|
|
||||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
|
||||||
{
|
|
||||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
|
||||||
U32 const nbBits = DInfo.nbBits;
|
|
||||||
BYTE const symbol = DInfo.symbol;
|
|
||||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
|
||||||
|
|
||||||
DStatePtr->state = DInfo.newState + lowBits;
|
|
||||||
return symbol;
|
|
||||||
}
|
|
||||||
|
|
||||||
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
|
||||||
{
|
|
||||||
return DStatePtr->state == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#ifndef FSE_COMMONDEFS_ONLY
|
|
||||||
|
|
||||||
/* **************************************************************
|
|
||||||
* Tuning parameters
|
|
||||||
****************************************************************/
|
|
||||||
/*!MEMORY_USAGE :
|
|
||||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
|
||||||
* Increasing memory usage improves compression ratio
|
|
||||||
* Reduced memory usage can improve speed, due to cache effect
|
|
||||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
|
||||||
#define FSE_MAX_MEMORY_USAGE 14
|
|
||||||
#define FSE_DEFAULT_MEMORY_USAGE 13
|
|
||||||
|
|
||||||
/*!FSE_MAX_SYMBOL_VALUE :
|
|
||||||
* Maximum symbol value authorized.
|
|
||||||
* Required for proper stack allocation */
|
|
||||||
#define FSE_MAX_SYMBOL_VALUE 255
|
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
|
||||||
* template functions type & suffix
|
|
||||||
****************************************************************/
|
|
||||||
#define FSE_FUNCTION_TYPE BYTE
|
|
||||||
#define FSE_FUNCTION_EXTENSION
|
|
||||||
#define FSE_DECODE_TYPE FSE_decode_t
|
|
||||||
|
|
||||||
|
|
||||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
|
||||||
|
|
||||||
|
|
||||||
/* ***************************************************************
|
|
||||||
* Constants
|
|
||||||
*****************************************************************/
|
|
||||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
|
||||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
|
||||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
|
||||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
|
||||||
#define FSE_MIN_TABLELOG 5
|
|
||||||
|
|
||||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
|
||||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
|
||||||
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif /* FSE_STATIC_H */
|
|
142
lib/common/huf.h
142
lib/common/huf.h
@ -39,13 +39,11 @@ extern "C" {
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
/* *** Dependencies *** */
|
||||||
* Dependency
|
|
||||||
******************************************/
|
|
||||||
#include <stddef.h> /* size_t */
|
#include <stddef.h> /* size_t */
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
/*-****************************************
|
||||||
* HUF simple functions
|
* HUF simple functions
|
||||||
******************************************/
|
******************************************/
|
||||||
size_t HUF_compress(void* dst, size_t dstCapacity,
|
size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||||
@ -54,17 +52,17 @@ size_t HUF_decompress(void* dst, size_t dstSize,
|
|||||||
const void* cSrc, size_t cSrcSize);
|
const void* cSrc, size_t cSrcSize);
|
||||||
/*
|
/*
|
||||||
HUF_compress() :
|
HUF_compress() :
|
||||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
||||||
'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize).
|
'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
||||||
Note : srcSize must be <= 128 KB
|
Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB
|
||||||
@return : size of compressed data (<= dstCapacity)
|
@return : size of compressed data (<= `dstCapacity`)
|
||||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||||
if return == 1, srcData is a single repeated byte symbol (RLE compression).
|
if return == 1, srcData is a single repeated byte symbol (RLE compression).
|
||||||
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||||
|
|
||||||
HUF_decompress() :
|
HUF_decompress() :
|
||||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||||
into already allocated destination buffer 'dst', of size 'dstSize'.
|
into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||||
`dstSize` : must be the **exact** size of original (uncompressed) data.
|
`dstSize` : must be the **exact** size of original (uncompressed) data.
|
||||||
Note : in contrast with FSE, HUF_decompress can regenerate
|
Note : in contrast with FSE, HUF_decompress can regenerate
|
||||||
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||||
@ -78,19 +76,137 @@ HUF_decompress() :
|
|||||||
* Tool functions
|
* Tool functions
|
||||||
******************************************/
|
******************************************/
|
||||||
#define HUF_BLOCKSIZE_MAX (128 * 1024)
|
#define HUF_BLOCKSIZE_MAX (128 * 1024)
|
||||||
size_t HUF_compressBound(size_t size); /**< maximum compressed size */
|
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
||||||
|
|
||||||
/* Error Management */
|
/* Error Management */
|
||||||
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
||||||
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
/* *** Advanced function *** */
|
||||||
* Advanced functions
|
|
||||||
******************************************/
|
/** HUF_compress2() :
|
||||||
|
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
|
||||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef HUF_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
/* *** Dependencies *** */
|
||||||
|
#include "mem.h" /* U32 */
|
||||||
|
|
||||||
|
|
||||||
|
/* *** Constants *** */
|
||||||
|
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||||
|
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||||
|
#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */
|
||||||
|
#define HUF_SYMBOLVALUE_MAX 255
|
||||||
|
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
||||||
|
# error "HUF_TABLELOG_MAX is too large !"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
/* ****************************************
|
||||||
|
* Static allocation
|
||||||
|
******************************************/
|
||||||
|
/* HUF buffer bounds */
|
||||||
|
#define HUF_CTABLEBOUND 129
|
||||||
|
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||||
|
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||||
|
|
||||||
|
/* static allocation of HUF's Compression Table */
|
||||||
|
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||||
|
U32 name##hb[maxSymbolValue+1]; \
|
||||||
|
void* name##hv = &(name##hb); \
|
||||||
|
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||||
|
|
||||||
|
/* static allocation of HUF's DTable */
|
||||||
|
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
|
||||||
|
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||||
|
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||||
|
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||||
|
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||||
|
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
|
||||||
|
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
|
||||||
|
|
||||||
|
|
||||||
|
/* ****************************************
|
||||||
|
* Advanced decompression functions
|
||||||
|
******************************************/
|
||||||
|
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||||
|
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
||||||
|
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||||
|
|
||||||
|
|
||||||
|
/* ****************************************
|
||||||
|
* HUF detailed API
|
||||||
|
******************************************/
|
||||||
|
/*!
|
||||||
|
HUF_compress() does the following:
|
||||||
|
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
||||||
|
2. (optional) refine tableLog using HUF_optimalTableLog()
|
||||||
|
3. build Huffman table from count using HUF_buildCTable()
|
||||||
|
4. save Huffman table to memory buffer using HUF_writeCTable()
|
||||||
|
5. encode the data stream using HUF_compress4X_usingCTable()
|
||||||
|
|
||||||
|
The following API allows targeting specific sub-functions for advanced tasks.
|
||||||
|
For example, it's possible to compress several blocks using the same 'CTable',
|
||||||
|
or to save and regenerate 'CTable' using external methods.
|
||||||
|
*/
|
||||||
|
/* FSE_count() : find it within "fse.h" */
|
||||||
|
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||||
|
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
||||||
|
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
||||||
|
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||||
|
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||||
|
|
||||||
|
|
||||||
|
/*!
|
||||||
|
HUF_decompress() does the following:
|
||||||
|
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
|
||||||
|
2. build Huffman table from save, using HUF_readDTableXn()
|
||||||
|
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
|
||||||
|
*/
|
||||||
|
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
|
||||||
|
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
|
||||||
|
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||||
|
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||||
|
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||||
|
|
||||||
|
|
||||||
|
/* single stream variants */
|
||||||
|
|
||||||
|
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||||
|
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||||
|
|
||||||
|
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||||
|
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||||
|
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||||
|
|
||||||
|
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||||
|
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||||
|
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||||
|
|
||||||
|
|
||||||
|
/*! HUF_readStats() :
|
||||||
|
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||||
|
`huffWeight` is destination buffer.
|
||||||
|
@return : size read from `src` , or an error Code .
|
||||||
|
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
||||||
|
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||||
|
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||||
|
const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
/** HUF_readCTable() :
|
||||||
|
* Loading a CTable saved with HUF_writeCTable() */
|
||||||
|
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* HUF_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -1,232 +0,0 @@
|
|||||||
/* ******************************************************************
|
|
||||||
Huffman codec, part of New Generation Entropy library
|
|
||||||
header file, for static linking only
|
|
||||||
Copyright (C) 2013-2016, Yann Collet
|
|
||||||
|
|
||||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
* Redistributions in binary form must reproduce the above
|
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
|
||||||
in the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
You can contact the author at :
|
|
||||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
|
||||||
****************************************************************** */
|
|
||||||
#ifndef HUF_STATIC_H
|
|
||||||
#define HUF_STATIC_H
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
|
||||||
* Dependency
|
|
||||||
******************************************/
|
|
||||||
#include "huf.h"
|
|
||||||
#include "fse.h"
|
|
||||||
#include "bitstream.h"
|
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
|
||||||
* Static allocation
|
|
||||||
******************************************/
|
|
||||||
/* HUF buffer bounds */
|
|
||||||
#define HUF_CTABLEBOUND 129
|
|
||||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
|
||||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
|
||||||
|
|
||||||
/* static allocation of HUF's Compression Table */
|
|
||||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
|
||||||
U32 name##hb[maxSymbolValue+1]; \
|
|
||||||
void* name##hv = &(name##hb); \
|
|
||||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
|
||||||
|
|
||||||
/* static allocation of HUF's DTable */
|
|
||||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
|
|
||||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
|
||||||
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
|
||||||
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
|
||||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
|
||||||
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
|
|
||||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
|
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
|
||||||
* Advanced decompression functions
|
|
||||||
******************************************/
|
|
||||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
|
||||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
|
||||||
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
|
||||||
|
|
||||||
|
|
||||||
/* ****************************************
|
|
||||||
* HUF detailed API
|
|
||||||
******************************************/
|
|
||||||
/*!
|
|
||||||
HUF_compress() does the following:
|
|
||||||
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
|
||||||
2. (optional) refine tableLog using HUF_optimalTableLog()
|
|
||||||
3. build Huffman table from count using HUF_buildCTable()
|
|
||||||
4. save Huffman table to memory buffer using HUF_writeCTable()
|
|
||||||
5. encode the data stream using HUF_compress4X_usingCTable()
|
|
||||||
|
|
||||||
The following API allows targeting specific sub-functions for advanced tasks.
|
|
||||||
For example, it's possible to compress several blocks using the same 'CTable',
|
|
||||||
or to save and regenerate 'CTable' using external methods.
|
|
||||||
*/
|
|
||||||
/* FSE_count() : find it within "fse.h" */
|
|
||||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
|
||||||
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
|
||||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
|
||||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
|
||||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
|
||||||
|
|
||||||
|
|
||||||
/*!
|
|
||||||
HUF_decompress() does the following:
|
|
||||||
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
|
|
||||||
2. build Huffman table from save, using HUF_readDTableXn()
|
|
||||||
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
|
|
||||||
*/
|
|
||||||
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
|
|
||||||
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
|
|
||||||
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
|
||||||
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
|
||||||
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
|
||||||
|
|
||||||
|
|
||||||
/* single stream variants */
|
|
||||||
|
|
||||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
|
||||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
|
||||||
|
|
||||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
|
||||||
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
|
||||||
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
|
||||||
|
|
||||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
|
||||||
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
|
||||||
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
|
||||||
|
|
||||||
|
|
||||||
/* Loading a CTable saved with HUF_writeCTable() */
|
|
||||||
|
|
||||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
|
||||||
* Constants
|
|
||||||
****************************************************************/
|
|
||||||
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
|
||||||
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
|
||||||
#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */
|
|
||||||
#define HUF_SYMBOLVALUE_MAX 255
|
|
||||||
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
|
||||||
# error "HUF_TABLELOG_MAX is too large !"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
|
||||||
* Needed by zstd in both compression and decompression
|
|
||||||
****************************************************************/
|
|
||||||
/*! HUF_readStats() :
|
|
||||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
|
||||||
`huffWeight` is destination buffer.
|
|
||||||
@return : size read from `src`
|
|
||||||
*/
|
|
||||||
MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
|
||||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
|
||||||
const void* src, size_t srcSize)
|
|
||||||
{
|
|
||||||
U32 weightTotal;
|
|
||||||
const BYTE* ip = (const BYTE*) src;
|
|
||||||
size_t iSize = ip[0];
|
|
||||||
size_t oSize;
|
|
||||||
|
|
||||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
|
||||||
|
|
||||||
if (iSize >= 128) { /* special header */
|
|
||||||
if (iSize >= (242)) { /* RLE */
|
|
||||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
|
||||||
oSize = l[iSize-242];
|
|
||||||
memset(huffWeight, 1, hwSize);
|
|
||||||
iSize = 0;
|
|
||||||
}
|
|
||||||
else { /* Incompressible */
|
|
||||||
oSize = iSize - 127;
|
|
||||||
iSize = ((oSize+1)/2);
|
|
||||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
|
||||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
|
||||||
ip += 1;
|
|
||||||
{ U32 n;
|
|
||||||
for (n=0; n<oSize; n+=2) {
|
|
||||||
huffWeight[n] = ip[n/2] >> 4;
|
|
||||||
huffWeight[n+1] = ip[n/2] & 15;
|
|
||||||
} } } }
|
|
||||||
else { /* header compressed with FSE (normal case) */
|
|
||||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
|
||||||
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
|
||||||
if (FSE_isError(oSize)) return oSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* collect weight stats */
|
|
||||||
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
|
|
||||||
weightTotal = 0;
|
|
||||||
{ U32 n; for (n=0; n<oSize; n++) {
|
|
||||||
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
|
||||||
rankStats[huffWeight[n]]++;
|
|
||||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
|
||||||
} }
|
|
||||||
|
|
||||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
|
||||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
|
||||||
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
|
||||||
*tableLogPtr = tableLog;
|
|
||||||
/* determine last weight */
|
|
||||||
{ U32 const total = 1 << tableLog;
|
|
||||||
U32 const rest = total - weightTotal;
|
|
||||||
U32 const verif = 1 << BIT_highbit32(rest);
|
|
||||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
|
||||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
|
||||||
huffWeight[oSize] = (BYTE)lastWeight;
|
|
||||||
rankStats[lastWeight]++;
|
|
||||||
} }
|
|
||||||
|
|
||||||
/* check tree construction validity */
|
|
||||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
|
||||||
|
|
||||||
/* results */
|
|
||||||
*nbSymbolsPtr = (U32)(oSize+1);
|
|
||||||
return iSize+1;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif /* HUF_STATIC_H */
|
|
@ -28,8 +28,8 @@
|
|||||||
You can contact the author at :
|
You can contact the author at :
|
||||||
- zstd homepage : http://www.zstd.net/
|
- zstd homepage : http://www.zstd.net/
|
||||||
*/
|
*/
|
||||||
#ifndef ZSTD_BUFFERED_H
|
#ifndef ZSTD_BUFFERED_H_23987
|
||||||
#define ZSTD_BUFFERED_H
|
#define ZSTD_BUFFERED_H_23987
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
extern "C" {
|
extern "C" {
|
||||||
@ -160,8 +160,40 @@ ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
|
|||||||
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
|
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef ZBUFF_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
/* ====================================================================================
|
||||||
|
* The definitions in this section are considered experimental.
|
||||||
|
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||||
|
* They are provided for advanced usages.
|
||||||
|
* Use them only in association with static linking.
|
||||||
|
* ==================================================================================== */
|
||||||
|
|
||||||
|
/*--- Dependency ---*/
|
||||||
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
||||||
|
#include "zstd.h"
|
||||||
|
|
||||||
|
|
||||||
|
/*--- External memory ---*/
|
||||||
|
/*! ZBUFF_createCCtx_advanced() :
|
||||||
|
* Create a ZBUFF compression context using external alloc and free functions */
|
||||||
|
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
/*! ZBUFF_createDCtx_advanced() :
|
||||||
|
* Create a ZBUFF decompression context using external alloc and free functions */
|
||||||
|
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
|
||||||
|
/*--- Advanced Streaming function ---*/
|
||||||
|
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||||
|
const void* dict, size_t dictSize,
|
||||||
|
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||||
|
|
||||||
|
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#endif /* ZSTD_BUFFERED_H */
|
#endif /* ZSTD_BUFFERED_H_23987 */
|
||||||
|
@ -1,84 +0,0 @@
|
|||||||
/*
|
|
||||||
zstd - buffered version of compression library
|
|
||||||
experimental complementary API, for static linking only
|
|
||||||
Copyright (C) 2015-2016, Yann Collet.
|
|
||||||
|
|
||||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
* Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
* Redistributions in binary form must reproduce the above
|
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
|
||||||
in the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
You can contact the author at :
|
|
||||||
- zstd homepage : http://www.zstd.net
|
|
||||||
*/
|
|
||||||
#ifndef ZSTD_BUFFERED_STATIC_H
|
|
||||||
#define ZSTD_BUFFERED_STATIC_H
|
|
||||||
|
|
||||||
/* The objects defined into this file should be considered experimental.
|
|
||||||
* They are not labelled stable, as their prototype may change in the future.
|
|
||||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* *************************************
|
|
||||||
* Includes
|
|
||||||
***************************************/
|
|
||||||
#include "zstd_static.h" /* ZSTD_parameters */
|
|
||||||
#include "zbuff.h"
|
|
||||||
|
|
||||||
#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b))
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Advanced functions
|
|
||||||
***************************************/
|
|
||||||
/*! ZBUFF_createCCtx_advanced() :
|
|
||||||
* Create a ZBUFF compression context using external alloc and free functions */
|
|
||||||
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
|
|
||||||
|
|
||||||
/*! ZBUFF_createDCtx_advanced() :
|
|
||||||
* Create a ZBUFF decompression context using external alloc and free functions */
|
|
||||||
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
|
||||||
|
|
||||||
|
|
||||||
/* *************************************
|
|
||||||
* Advanced Streaming functions
|
|
||||||
***************************************/
|
|
||||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
|
|
||||||
const void* dict, size_t dictSize,
|
|
||||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
|
||||||
|
|
||||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
|
||||||
{
|
|
||||||
size_t length = ZBUFF_MIN(dstCapacity, srcSize);
|
|
||||||
memcpy(dst, src, length);
|
|
||||||
return length;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif /* ZSTD_BUFFERED_STATIC_H */
|
|
@ -29,8 +29,8 @@
|
|||||||
You can contact the author at :
|
You can contact the author at :
|
||||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||||
*/
|
*/
|
||||||
#ifndef ZSTD_H
|
#ifndef ZSTD_H_235446
|
||||||
#define ZSTD_H
|
#define ZSTD_H_235446
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
extern "C" {
|
extern "C" {
|
||||||
@ -60,8 +60,8 @@ extern "C" {
|
|||||||
* Version
|
* Version
|
||||||
***************************************/
|
***************************************/
|
||||||
#define ZSTD_VERSION_MAJOR 0
|
#define ZSTD_VERSION_MAJOR 0
|
||||||
#define ZSTD_VERSION_MINOR 6
|
#define ZSTD_VERSION_MINOR 7
|
||||||
#define ZSTD_VERSION_RELEASE 2
|
#define ZSTD_VERSION_RELEASE 0
|
||||||
|
|
||||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||||
#define ZSTD_QUOTE(str) #str
|
#define ZSTD_QUOTE(str) #str
|
||||||
@ -148,8 +148,267 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|||||||
const void* dict,size_t dictSize);
|
const void* dict,size_t dictSize);
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef ZSTD_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
/* ====================================================================================
|
||||||
|
* The definitions in this section are considered experimental.
|
||||||
|
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||||
|
* They are provided for advanced usages.
|
||||||
|
* Use them only in association with static linking.
|
||||||
|
* ==================================================================================== */
|
||||||
|
|
||||||
|
/*--- Dependency ---*/
|
||||||
|
#include "mem.h"
|
||||||
|
|
||||||
|
|
||||||
|
/*--- Constants ---*/
|
||||||
|
#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */
|
||||||
|
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||||
|
|
||||||
|
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
|
||||||
|
#define ZSTD_WINDOWLOG_MIN 18
|
||||||
|
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||||
|
#define ZSTD_CHAINLOG_MIN 4
|
||||||
|
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
|
||||||
|
#define ZSTD_HASHLOG_MIN 12
|
||||||
|
#define ZSTD_HASHLOG3_MAX 17
|
||||||
|
#define ZSTD_HASHLOG3_MIN 15
|
||||||
|
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||||
|
#define ZSTD_SEARCHLOG_MIN 1
|
||||||
|
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||||
|
#define ZSTD_SEARCHLENGTH_MIN 3
|
||||||
|
#define ZSTD_TARGETLENGTH_MIN 4
|
||||||
|
#define ZSTD_TARGETLENGTH_MAX 999
|
||||||
|
|
||||||
|
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||||
|
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||||
|
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||||
|
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||||
|
|
||||||
|
|
||||||
|
/*--- Types ---*/
|
||||||
|
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
|
||||||
|
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
||||||
|
U32 hashLog; /* dispatch table : larger == faster, more memory */
|
||||||
|
U32 searchLog; /* nb of searches : larger == more compression, slower */
|
||||||
|
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
|
||||||
|
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
|
||||||
|
ZSTD_strategy strategy;
|
||||||
|
} ZSTD_compressionParameters;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
||||||
|
U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
|
||||||
|
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
|
||||||
|
} ZSTD_frameParameters;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ZSTD_compressionParameters cParams;
|
||||||
|
ZSTD_frameParameters fParams;
|
||||||
|
} ZSTD_parameters;
|
||||||
|
|
||||||
|
/* custom memory allocation functions */
|
||||||
|
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||||
|
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||||
|
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Advanced functions
|
||||||
|
***************************************/
|
||||||
|
/*! ZSTD_createCCtx_advanced() :
|
||||||
|
* Create a ZSTD compression context using external alloc and free functions */
|
||||||
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
/*! ZSTD_createDCtx_advanced() :
|
||||||
|
* Create a ZSTD decompression context using external alloc and free functions */
|
||||||
|
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
||||||
|
|
||||||
|
/*! ZSTD_getCParams() :
|
||||||
|
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||||
|
* `srcSize` value is optional, select 0 if not known */
|
||||||
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||||
|
|
||||||
|
/*! ZSTD_checkParams() :
|
||||||
|
* Ensure param values remain within authorized range */
|
||||||
|
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||||
|
|
||||||
|
/*! ZSTD_adjustParams() :
|
||||||
|
* optimize params for a given `srcSize` and `dictSize`.
|
||||||
|
* both values are optional, select `0` if unknown. */
|
||||||
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
|
||||||
|
|
||||||
|
/*! ZSTD_compress_advanced() :
|
||||||
|
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||||
|
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||||
|
void* dst, size_t dstCapacity,
|
||||||
|
const void* src, size_t srcSize,
|
||||||
|
const void* dict,size_t dictSize,
|
||||||
|
ZSTD_parameters params);
|
||||||
|
|
||||||
|
/*! ZSTD_compress_usingPreparedDCtx() :
|
||||||
|
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
||||||
|
* It avoids reloading the dictionary each time.
|
||||||
|
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
||||||
|
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
||||||
|
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
|
||||||
|
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||||
|
void* dst, size_t dstCapacity,
|
||||||
|
const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
/*- Advanced Decompression functions -*/
|
||||||
|
|
||||||
|
/*! ZSTD_decompress_usingPreparedDCtx() :
|
||||||
|
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||||
|
* It avoids reloading the dictionary each time.
|
||||||
|
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
|
||||||
|
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||||
|
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
|
||||||
|
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
|
||||||
|
void* dst, size_t dstCapacity,
|
||||||
|
const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
|
||||||
|
/* **************************************
|
||||||
|
* Streaming functions (direct mode)
|
||||||
|
****************************************/
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
|
||||||
|
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
||||||
|
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
|
||||||
|
|
||||||
|
/*
|
||||||
|
Streaming compression, synchronous mode (bufferless)
|
||||||
|
|
||||||
|
A ZSTD_CCtx object is required to track streaming operations.
|
||||||
|
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
|
||||||
|
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||||
|
|
||||||
|
Start by initializing a context.
|
||||||
|
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
|
||||||
|
or ZSTD_compressBegin_advanced(), for finer parameter control.
|
||||||
|
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
|
||||||
|
|
||||||
|
Then, consume your input using ZSTD_compressContinue().
|
||||||
|
The interface is synchronous, so all input will be consumed and produce a compressed output.
|
||||||
|
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
|
||||||
|
Worst case evaluation is provided by ZSTD_compressBound().
|
||||||
|
|
||||||
|
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
|
||||||
|
Without the epilogue, frames will be considered incomplete by decoder.
|
||||||
|
|
||||||
|
You can then reuse ZSTD_CCtx to compress some new frame.
|
||||||
|
*/
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U64 frameContentSize;
|
||||||
|
U32 windowSize;
|
||||||
|
U32 dictID;
|
||||||
|
U32 checksumFlag;
|
||||||
|
} ZSTD_frameParams;
|
||||||
|
|
||||||
|
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||||
|
|
||||||
|
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||||
|
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||||
|
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||||
|
|
||||||
|
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||||
|
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
/*
|
||||||
|
Streaming decompression, direct mode (bufferless)
|
||||||
|
|
||||||
|
A ZSTD_DCtx object is required to track streaming operations.
|
||||||
|
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||||
|
A ZSTD_DCtx object can be re-used multiple times.
|
||||||
|
|
||||||
|
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
|
||||||
|
It can provide the minimum size of rolling buffer required to properly decompress data,
|
||||||
|
and optionally the final size of uncompressed content.
|
||||||
|
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||||
|
Frame parameters are extracted from the beginning of compressed frame.
|
||||||
|
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
|
||||||
|
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||||
|
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
||||||
|
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||||
|
errorCode, which can be tested using ZSTD_isError()
|
||||||
|
|
||||||
|
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
|
||||||
|
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
|
||||||
|
|
||||||
|
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||||
|
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||||
|
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
|
||||||
|
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
||||||
|
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
||||||
|
|
||||||
|
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||||
|
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||||
|
|
||||||
|
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||||
|
Context can then be reset to start a new decompression.
|
||||||
|
|
||||||
|
Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
|
||||||
|
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following:
|
||||||
|
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||||
|
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||||
|
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||||
|
For skippable frames ZSTD_decompressContinue() always returns 0.
|
||||||
|
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||||
|
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
/* **************************************
|
||||||
|
* Block functions
|
||||||
|
****************************************/
|
||||||
|
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||||
|
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||||
|
|
||||||
|
A few rules to respect :
|
||||||
|
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
|
||||||
|
- Compressing or decompressing requires a context structure
|
||||||
|
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||||
|
- It is necessary to init context before starting
|
||||||
|
+ compression : ZSTD_compressBegin()
|
||||||
|
+ decompression : ZSTD_decompressBegin()
|
||||||
|
+ variants _usingDict() are also allowed
|
||||||
|
+ copyCCtx() and copyDCtx() work too
|
||||||
|
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
||||||
|
In which case, nothing is produced into `dst`.
|
||||||
|
+ User must test for such outcome and deal directly with uncompressed data
|
||||||
|
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
|
||||||
|
*/
|
||||||
|
|
||||||
|
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||||
|
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
|
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Error management
|
||||||
|
***************************************/
|
||||||
|
#include "error_public.h"
|
||||||
|
/*! ZSTD_getErrorCode() :
|
||||||
|
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||||
|
which can be used to compare directly with enum list published into "error_public.h" */
|
||||||
|
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||||
|
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* ZSTD_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#endif /* ZSTD_H */
|
#endif /* ZSTD_H_235446 */
|
||||||
|
@ -33,8 +33,10 @@
|
|||||||
/*-*************************************
|
/*-*************************************
|
||||||
* Dependencies
|
* Dependencies
|
||||||
***************************************/
|
***************************************/
|
||||||
|
#include <stdlib.h> /* malloc */
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */
|
#define ZSTD_STATIC_LINKING_ONLY
|
||||||
|
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
|
||||||
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
|
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
|
||||||
|
|
||||||
|
|
||||||
@ -70,3 +72,20 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c
|
|||||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||||
|
|
||||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
|
||||||
|
{
|
||||||
|
void* address = malloc(size);
|
||||||
|
(void)opaque;
|
||||||
|
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
void ZSTD_defaultFreeFunction(void* opaque, void* address)
|
||||||
|
{
|
||||||
|
(void)opaque;
|
||||||
|
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
|
||||||
|
free(address);
|
||||||
|
}
|
||||||
|
@ -37,7 +37,8 @@
|
|||||||
***************************************/
|
***************************************/
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
#include "zstd_static.h"
|
#define ZSTD_STATIC_LINKING_ONLY
|
||||||
|
#include "zstd.h"
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
@ -63,7 +64,7 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define ZSTD_OPT_NUM (1<<12)
|
#define ZSTD_OPT_NUM (1<<12)
|
||||||
#define ZSTD_DICT_MAGIC 0xEC30A436
|
#define ZSTD_DICT_MAGIC 0xEC30A437
|
||||||
|
|
||||||
#define ZSTD_REP_NUM 3
|
#define ZSTD_REP_NUM 3
|
||||||
#define ZSTD_REP_INIT ZSTD_REP_NUM
|
#define ZSTD_REP_INIT ZSTD_REP_NUM
|
||||||
@ -80,10 +81,11 @@
|
|||||||
#define BIT1 2
|
#define BIT1 2
|
||||||
#define BIT0 1
|
#define BIT0 1
|
||||||
|
|
||||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12
|
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
|
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||||
|
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||||
|
|
||||||
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
|
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||||
|
|
||||||
@ -160,28 +162,6 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
|
|||||||
while (op < oend);
|
while (op < oend);
|
||||||
}
|
}
|
||||||
|
|
||||||
MEM_STATIC unsigned ZSTD_highbit(U32 val)
|
|
||||||
{
|
|
||||||
# if defined(_MSC_VER) /* Visual */
|
|
||||||
unsigned long r=0;
|
|
||||||
_BitScanReverse(&r, val);
|
|
||||||
return (unsigned)r;
|
|
||||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
|
||||||
return 31 - __builtin_clz(val);
|
|
||||||
# else /* Software version */
|
|
||||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
|
||||||
U32 v = val;
|
|
||||||
int r;
|
|
||||||
v |= v >> 1;
|
|
||||||
v |= v >> 2;
|
|
||||||
v |= v >> 4;
|
|
||||||
v |= v >> 8;
|
|
||||||
v |= v >> 16;
|
|
||||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
|
||||||
return r;
|
|
||||||
# endif
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/*-*******************************************
|
/*-*******************************************
|
||||||
* Private interfaces
|
* Private interfaces
|
||||||
@ -209,7 +189,7 @@ typedef struct {
|
|||||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
|
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
|
||||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
|
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
|
||||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||||
#endif // #if ZSTD_OPT_DEBUG == 3
|
#endif /* #if ZSTD_OPT_DEBUG == 3 */
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void* buffer;
|
void* buffer;
|
||||||
@ -252,6 +232,11 @@ typedef struct {
|
|||||||
|
|
||||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||||
|
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
|
||||||
|
|
||||||
|
/* custom memory allocation functions */
|
||||||
|
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
|
||||||
|
void ZSTD_defaultFreeFunction(void* opaque, void* address);
|
||||||
|
static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
|
||||||
|
|
||||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||||
|
@ -1,284 +0,0 @@
|
|||||||
/*
|
|
||||||
zstd - standard compression library
|
|
||||||
Header File for static linking only
|
|
||||||
Copyright (C) 2014-2016, Yann Collet.
|
|
||||||
|
|
||||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
* Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
* Redistributions in binary form must reproduce the above
|
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
|
||||||
in the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
You can contact the author at :
|
|
||||||
- zstd homepage : http://www.zstd.net
|
|
||||||
*/
|
|
||||||
#ifndef ZSTD_STATIC_H
|
|
||||||
#define ZSTD_STATIC_H
|
|
||||||
|
|
||||||
/* The prototypes defined within this file are considered experimental.
|
|
||||||
* They should not be used in the context DLL as they may change in the future.
|
|
||||||
* Prefer static linking if you need them, to control breaking version changes issues.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Dependencies
|
|
||||||
***************************************/
|
|
||||||
#include "zstd.h"
|
|
||||||
#include "mem.h"
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Constants
|
|
||||||
***************************************/
|
|
||||||
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Types
|
|
||||||
***************************************/
|
|
||||||
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
|
|
||||||
#define ZSTD_WINDOWLOG_MIN 18
|
|
||||||
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
|
||||||
#define ZSTD_CHAINLOG_MIN 4
|
|
||||||
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
|
|
||||||
#define ZSTD_HASHLOG_MIN 12
|
|
||||||
#define ZSTD_HASHLOG3_MAX 17
|
|
||||||
#define ZSTD_HASHLOG3_MIN 15
|
|
||||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
|
||||||
#define ZSTD_SEARCHLOG_MIN 1
|
|
||||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
|
||||||
#define ZSTD_SEARCHLENGTH_MIN 3
|
|
||||||
#define ZSTD_TARGETLENGTH_MIN 4
|
|
||||||
#define ZSTD_TARGETLENGTH_MAX 999
|
|
||||||
|
|
||||||
/* from faster to stronger */
|
|
||||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
|
|
||||||
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
|
||||||
U32 hashLog; /* dispatch table : larger == faster, more memory */
|
|
||||||
U32 searchLog; /* nb of searches : larger == more compression, slower */
|
|
||||||
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
|
|
||||||
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
|
|
||||||
ZSTD_strategy strategy;
|
|
||||||
} ZSTD_compressionParameters;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
|
||||||
} ZSTD_frameParameters;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
ZSTD_compressionParameters cParams;
|
|
||||||
ZSTD_frameParameters fParams;
|
|
||||||
} ZSTD_parameters;
|
|
||||||
|
|
||||||
typedef void* (*ZSTD_allocFunction) (size_t size);
|
|
||||||
typedef void (*ZSTD_freeFunction) (void* address);
|
|
||||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem;
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Advanced functions
|
|
||||||
***************************************/
|
|
||||||
/*! ZSTD_createCCtx_advanced() :
|
|
||||||
* Create a ZSTD compression context using external alloc and free functions */
|
|
||||||
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
|
||||||
|
|
||||||
/*! ZSTD_createDCtx_advanced() :
|
|
||||||
* Create a ZSTD decompression context using external alloc and free functions */
|
|
||||||
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
|
||||||
|
|
||||||
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
|
||||||
|
|
||||||
/*! ZSTD_getCParams() :
|
|
||||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
|
||||||
* `srcSize` value is optional, select 0 if not known */
|
|
||||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
|
||||||
|
|
||||||
/*! ZSTD_checkParams() :
|
|
||||||
* Ensure param values remain within authorized range */
|
|
||||||
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|
||||||
|
|
||||||
/*! ZSTD_adjustParams() :
|
|
||||||
* optimize params for a given `srcSize` and `dictSize`.
|
|
||||||
* both values are optional, select `0` if unknown. */
|
|
||||||
ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize);
|
|
||||||
|
|
||||||
/*! ZSTD_compress_advanced() :
|
|
||||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
|
||||||
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
const void* src, size_t srcSize,
|
|
||||||
const void* dict,size_t dictSize,
|
|
||||||
ZSTD_parameters params);
|
|
||||||
|
|
||||||
/*! ZSTD_compress_usingPreparedDCtx() :
|
|
||||||
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
|
||||||
* It avoids reloading the dictionary each time.
|
|
||||||
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
|
||||||
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
|
||||||
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
|
|
||||||
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
/*- Advanced Decompression functions -*/
|
|
||||||
|
|
||||||
/*! ZSTD_decompress_usingPreparedDCtx() :
|
|
||||||
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
|
||||||
* It avoids reloading the dictionary each time.
|
|
||||||
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
|
|
||||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
|
|
||||||
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
|
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
|
|
||||||
/* **************************************
|
|
||||||
* Streaming functions (direct mode)
|
|
||||||
****************************************/
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
|
|
||||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
|
|
||||||
|
|
||||||
/*
|
|
||||||
Streaming compression, synchronous mode (bufferless)
|
|
||||||
|
|
||||||
A ZSTD_CCtx object is required to track streaming operations.
|
|
||||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
|
|
||||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
|
||||||
|
|
||||||
Start by initializing a context.
|
|
||||||
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
|
|
||||||
or ZSTD_compressBegin_advanced(), for finer parameter control.
|
|
||||||
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
|
|
||||||
|
|
||||||
Then, consume your input using ZSTD_compressContinue().
|
|
||||||
The interface is synchronous, so all input will be consumed and produce a compressed output.
|
|
||||||
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
|
|
||||||
Worst case evaluation is provided by ZSTD_compressBound().
|
|
||||||
|
|
||||||
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
|
|
||||||
Without the epilogue, frames will be considered incomplete by decoder.
|
|
||||||
|
|
||||||
You can then reuse ZSTD_CCtx to compress some new frame.
|
|
||||||
*/
|
|
||||||
|
|
||||||
typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
|
|
||||||
|
|
||||||
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
|
|
||||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
|
||||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
|
||||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
|
||||||
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
/*
|
|
||||||
Streaming decompression, direct mode (bufferless)
|
|
||||||
|
|
||||||
A ZSTD_DCtx object is required to track streaming operations.
|
|
||||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
|
||||||
A ZSTD_DCtx object can be re-used multiple times.
|
|
||||||
|
|
||||||
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
|
|
||||||
It can provide the minimum size of rolling buffer required to properly decompress data,
|
|
||||||
and optionally the final size of uncompressed content.
|
|
||||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
|
||||||
Frame parameters are extracted from the beginning of compressed frame.
|
|
||||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
|
|
||||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
|
||||||
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
|
||||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
|
||||||
errorCode, which can be tested using ZSTD_isError()
|
|
||||||
|
|
||||||
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
|
|
||||||
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
|
|
||||||
|
|
||||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
|
||||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
|
||||||
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
|
|
||||||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
|
||||||
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
|
||||||
|
|
||||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
|
|
||||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
|
||||||
|
|
||||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
|
||||||
Context can then be reset to start a new decompression.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/* **************************************
|
|
||||||
* Block functions
|
|
||||||
****************************************/
|
|
||||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
|
||||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
|
||||||
|
|
||||||
A few rules to respect :
|
|
||||||
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
|
|
||||||
- Compressing or decompressing requires a context structure
|
|
||||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
|
||||||
- It is necessary to init context before starting
|
|
||||||
+ compression : ZSTD_compressBegin()
|
|
||||||
+ decompression : ZSTD_decompressBegin()
|
|
||||||
+ variants _usingDict() are also allowed
|
|
||||||
+ copyCCtx() and copyDCtx() work too
|
|
||||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
|
||||||
In which case, nothing is produced into `dst`.
|
|
||||||
+ User must test for such outcome and deal directly with uncompressed data
|
|
||||||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
|
|
||||||
*/
|
|
||||||
|
|
||||||
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
|
||||||
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Error management
|
|
||||||
***************************************/
|
|
||||||
#include "error_public.h"
|
|
||||||
/*! ZSTD_getErrorCode() :
|
|
||||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
|
||||||
which can be used to compare directly with enum list published into "error_public.h" */
|
|
||||||
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
|
||||||
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif /* ZSTD_STATIC_H */
|
|
@ -57,7 +57,8 @@
|
|||||||
#include <string.h> /* memcpy, memset */
|
#include <string.h> /* memcpy, memset */
|
||||||
#include <stdio.h> /* printf (debug) */
|
#include <stdio.h> /* printf (debug) */
|
||||||
#include "bitstream.h"
|
#include "bitstream.h"
|
||||||
#include "fse_static.h"
|
#define FSE_STATIC_LINKING_ONLY
|
||||||
|
#include "fse.h"
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
/* **************************************************************
|
||||||
|
@ -59,12 +59,13 @@
|
|||||||
/* **************************************************************
|
/* **************************************************************
|
||||||
* Includes
|
* Includes
|
||||||
****************************************************************/
|
****************************************************************/
|
||||||
#include <stdlib.h> /* malloc, free, qsort */
|
|
||||||
#include <string.h> /* memcpy, memset */
|
#include <string.h> /* memcpy, memset */
|
||||||
#include <stdio.h> /* printf (debug) */
|
#include <stdio.h> /* printf (debug) */
|
||||||
#include "huf_static.h"
|
|
||||||
#include "bitstream.h"
|
#include "bitstream.h"
|
||||||
#include "fse_static.h" /* header compression */
|
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
||||||
|
#include "fse.h" /* header compression */
|
||||||
|
#define HUF_STATIC_LINKING_ONLY
|
||||||
|
#include "huf.h"
|
||||||
|
|
||||||
|
|
||||||
/* **************************************************************
|
/* **************************************************************
|
||||||
|
@ -35,9 +35,9 @@
|
|||||||
***************************************/
|
***************************************/
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
|
||||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
#define ZBUFF_STATIC_LINKING_ONLY
|
||||||
#include "zbuff_static.h"
|
#include "zbuff.h"
|
||||||
|
|
||||||
|
|
||||||
/* *************************************
|
/* *************************************
|
||||||
@ -95,14 +95,12 @@ struct ZBUFF_CCtx_s {
|
|||||||
size_t outBuffContentSize;
|
size_t outBuffContentSize;
|
||||||
size_t outBuffFlushedSize;
|
size_t outBuffFlushedSize;
|
||||||
ZBUFF_cStage stage;
|
ZBUFF_cStage stage;
|
||||||
ZSTD_allocFunction customAlloc;
|
ZSTD_customMem customMem;
|
||||||
ZSTD_freeFunction customFree;
|
|
||||||
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
|
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
|
||||||
|
|
||||||
ZBUFF_CCtx* ZBUFF_createCCtx(void)
|
ZBUFF_CCtx* ZBUFF_createCCtx(void)
|
||||||
{
|
{
|
||||||
ZSTD_customMem customMem = { NULL, NULL };
|
return ZBUFF_createCCtx_advanced(defaultCustomMem);
|
||||||
return ZBUFF_createCCtx_advanced(customMem);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
|
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
|
||||||
@ -110,24 +108,17 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
|
|||||||
ZBUFF_CCtx* zbc;
|
ZBUFF_CCtx* zbc;
|
||||||
|
|
||||||
if (!customMem.customAlloc && !customMem.customFree)
|
if (!customMem.customAlloc && !customMem.customFree)
|
||||||
{
|
customMem = defaultCustomMem;
|
||||||
zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx));
|
|
||||||
if (zbc==NULL) return NULL;
|
|
||||||
zbc->customAlloc = malloc;
|
|
||||||
zbc->customFree = free;
|
|
||||||
zbc->zc = ZSTD_createCCtx();
|
|
||||||
return zbc;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!customMem.customAlloc || !customMem.customFree)
|
if (!customMem.customAlloc || !customMem.customFree)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx));
|
zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
|
||||||
if (zbc==NULL) return NULL;
|
if (zbc==NULL) return NULL;
|
||||||
memset(zbc, 0, sizeof(ZBUFF_CCtx));
|
memset(zbc, 0, sizeof(ZBUFF_CCtx));
|
||||||
zbc->customAlloc = customMem.customAlloc;
|
memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||||
zbc->customFree = customMem.customFree;
|
|
||||||
zbc->zc = ZSTD_createCCtx_advanced(customMem);
|
zbc->zc = ZSTD_createCCtx_advanced(customMem);
|
||||||
|
if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
|
||||||
return zbc;
|
return zbc;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -135,9 +126,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
|
|||||||
{
|
{
|
||||||
if (zbc==NULL) return 0; /* support free on NULL */
|
if (zbc==NULL) return 0; /* support free on NULL */
|
||||||
ZSTD_freeCCtx(zbc->zc);
|
ZSTD_freeCCtx(zbc->zc);
|
||||||
zbc->customFree(zbc->inBuff);
|
if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
|
||||||
zbc->customFree(zbc->outBuff);
|
if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
|
||||||
zbc->customFree(zbc);
|
zbc->customMem.customFree(zbc->customMem.opaque, zbc);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -152,16 +143,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
|||||||
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||||
if (zbc->inBuffSize < neededInBuffSize) {
|
if (zbc->inBuffSize < neededInBuffSize) {
|
||||||
zbc->inBuffSize = neededInBuffSize;
|
zbc->inBuffSize = neededInBuffSize;
|
||||||
zbc->customFree(zbc->inBuff); /* should not be necessary */
|
zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
|
||||||
zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize);
|
zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
|
||||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||||
}
|
}
|
||||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
|
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize);
|
||||||
}
|
}
|
||||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||||
zbc->customFree(zbc->outBuff); /* should not be necessary */
|
zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
|
||||||
zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize);
|
zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
|
||||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -180,9 +171,8 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
|||||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||||
params.fParams.contentSizeFlag = 0;
|
|
||||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
|
||||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,6 +182,15 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* internal util function */
|
||||||
|
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
|
{
|
||||||
|
size_t const length = MIN(dstCapacity, srcSize);
|
||||||
|
memcpy(dst, src, length);
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* *** Compression *** */
|
/* *** Compression *** */
|
||||||
|
|
||||||
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||||
|
@ -51,12 +51,15 @@
|
|||||||
/*-*************************************
|
/*-*************************************
|
||||||
* Dependencies
|
* Dependencies
|
||||||
***************************************/
|
***************************************/
|
||||||
#include <stdlib.h> /* malloc */
|
|
||||||
#include <string.h> /* memset */
|
#include <string.h> /* memset */
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "fse_static.h"
|
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||||
#include "huf_static.h"
|
#include "xxhash.h" /* XXH_reset, update, digest */
|
||||||
#include "zstd_internal.h"
|
#define FSE_STATIC_LINKING_ONLY
|
||||||
|
#include "fse.h"
|
||||||
|
#define HUF_STATIC_LINKING_ONLY
|
||||||
|
#include "huf.h"
|
||||||
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
@ -70,6 +73,27 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
|
|||||||
***************************************/
|
***************************************/
|
||||||
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
|
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
|
||||||
|
|
||||||
|
static U32 ZSTD_highbit32(U32 val)
|
||||||
|
{
|
||||||
|
# if defined(_MSC_VER) /* Visual */
|
||||||
|
unsigned long r=0;
|
||||||
|
_BitScanReverse(&r, val);
|
||||||
|
return (unsigned)r;
|
||||||
|
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||||
|
return 31 - __builtin_clz(val);
|
||||||
|
# else /* Software version */
|
||||||
|
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||||
|
U32 v = val;
|
||||||
|
int r;
|
||||||
|
v |= v >> 1;
|
||||||
|
v |= v >> 2;
|
||||||
|
v |= v >> 4;
|
||||||
|
v |= v >> 8;
|
||||||
|
v |= v >> 16;
|
||||||
|
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||||
|
return r;
|
||||||
|
# endif
|
||||||
|
}
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
* Sequence storage
|
* Sequence storage
|
||||||
@ -99,12 +123,14 @@ struct ZSTD_CCtx_s
|
|||||||
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
||||||
U32 loadedDictEnd;
|
U32 loadedDictEnd;
|
||||||
U32 stage; /* 0: created; 1: init,dictLoad; 2:started */
|
U32 stage; /* 0: created; 1: init,dictLoad; 2:started */
|
||||||
|
U32 dictID;
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
void* workSpace;
|
void* workSpace;
|
||||||
size_t workSpaceSize;
|
size_t workSpaceSize;
|
||||||
size_t blockSize;
|
size_t blockSize;
|
||||||
ZSTD_allocFunction customAlloc;
|
U64 frameContentSize;
|
||||||
ZSTD_freeFunction customFree;
|
XXH64_state_t xxhState;
|
||||||
|
ZSTD_customMem customMem;
|
||||||
|
|
||||||
seqStore_t seqStore; /* sequences storage ptrs */
|
seqStore_t seqStore; /* sequences storage ptrs */
|
||||||
U32* hashTable;
|
U32* hashTable;
|
||||||
@ -119,8 +145,7 @@ struct ZSTD_CCtx_s
|
|||||||
|
|
||||||
ZSTD_CCtx* ZSTD_createCCtx(void)
|
ZSTD_CCtx* ZSTD_createCCtx(void)
|
||||||
{
|
{
|
||||||
ZSTD_customMem customMem = { NULL, NULL };
|
return ZSTD_createCCtx_advanced(defaultCustomMem);
|
||||||
return ZSTD_createCCtx_advanced(customMem);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
|
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
|
||||||
@ -128,31 +153,23 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
|
|||||||
ZSTD_CCtx* ctx;
|
ZSTD_CCtx* ctx;
|
||||||
|
|
||||||
if (!customMem.customAlloc && !customMem.customFree)
|
if (!customMem.customAlloc && !customMem.customFree)
|
||||||
{
|
customMem = defaultCustomMem;
|
||||||
ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
|
|
||||||
if (!ctx) return NULL;
|
|
||||||
|
|
||||||
ctx->customAlloc = malloc;
|
|
||||||
ctx->customFree = free;
|
|
||||||
return ctx;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!customMem.customAlloc || !customMem.customFree)
|
if (!customMem.customAlloc || !customMem.customFree)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx));
|
ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx));
|
||||||
if (!ctx) return NULL;
|
if (!ctx) return NULL;
|
||||||
|
|
||||||
memset(ctx, 0, sizeof(ZSTD_CCtx));
|
memset(ctx, 0, sizeof(ZSTD_CCtx));
|
||||||
ctx->customAlloc = customMem.customAlloc;
|
memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||||
ctx->customFree = customMem.customFree;
|
|
||||||
return ctx;
|
return ctx;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||||
{
|
{
|
||||||
cctx->customFree(cctx->workSpace);
|
if (cctx==NULL) return 0; /* support free on NULL */
|
||||||
cctx->customFree(cctx);
|
if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace);
|
||||||
|
cctx->customMem.customFree(cctx->customMem.opaque, cctx);
|
||||||
return 0; /* reserved as a potential error code in the future */
|
return 0; /* reserved as a potential error code in the future */
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -183,8 +200,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static unsigned ZSTD_highbit(U32 val);
|
|
||||||
|
|
||||||
/** ZSTD_checkCParams_advanced() :
|
/** ZSTD_checkCParams_advanced() :
|
||||||
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
|
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
|
||||||
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
|
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
|
||||||
@ -198,30 +213,32 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/** ZSTD_adjustParams() :
|
/** ZSTD_adjustCParams() :
|
||||||
optimize params for q given input (`srcSize` and `dictSize`).
|
optimize cPar for a given input (`srcSize` and `dictSize`).
|
||||||
mostly downsizing to reduce memory consumption and initialization.
|
mostly downsizing to reduce memory consumption and initialization.
|
||||||
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
||||||
but if both are 0, no optimization can be done.
|
but if both are 0, no optimization can be done.
|
||||||
Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
||||||
void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize)
|
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize)
|
||||||
{
|
{
|
||||||
if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */
|
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
|
||||||
|
|
||||||
/* resize params, to use less memory when necessary */
|
/* resize params, to use less memory when necessary */
|
||||||
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
||||||
U64 const rSize = srcSize + dictSize + minSrcSize;
|
U64 const rSize = srcSize + dictSize + minSrcSize;
|
||||||
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
||||||
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
|
U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
|
||||||
if (params->windowLog > srcLog) params->windowLog = srcLog;
|
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
|
||||||
} }
|
} }
|
||||||
if (params->hashLog > params->windowLog) params->hashLog = params->windowLog;
|
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
|
||||||
{ U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt);
|
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt);
|
||||||
U32 const maxChainLog = params->windowLog+btPlus;
|
U32 const maxChainLog = cPar.windowLog+btPlus;
|
||||||
if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
||||||
|
|
||||||
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||||
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
||||||
|
|
||||||
|
return cPar;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -229,6 +246,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
|||||||
{
|
{
|
||||||
ZSTD_CCtx* zc = ZSTD_createCCtx();
|
ZSTD_CCtx* zc = ZSTD_createCCtx();
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = cParams;
|
params.cParams = cParams;
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
|
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
|
||||||
@ -240,7 +258,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
|||||||
/*! ZSTD_resetCCtx_advanced() :
|
/*! ZSTD_resetCCtx_advanced() :
|
||||||
note : 'params' is expected to be validated */
|
note : 'params' is expected to be validated */
|
||||||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||||
ZSTD_parameters params, U32 reset)
|
ZSTD_parameters params, U64 frameContentSize, U32 reset)
|
||||||
{ /* note : params considered validated here */
|
{ /* note : params considered validated here */
|
||||||
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
|
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
|
||||||
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
|
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
|
||||||
@ -248,7 +266,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
|||||||
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
||||||
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
||||||
const size_t hSize = ((size_t)1) << params.cParams.hashLog;
|
const size_t hSize = ((size_t)1) << params.cParams.hashLog;
|
||||||
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 :
|
||||||
|
( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX :
|
||||||
|
((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) );
|
||||||
|
const size_t h3Size = ((size_t)1) << hashLog3;
|
||||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||||
|
|
||||||
/* Check if workSpace is large enough, alloc a new one if needed */
|
/* Check if workSpace is large enough, alloc a new one if needed */
|
||||||
@ -257,17 +278,19 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
|||||||
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
|
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
|
||||||
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
|
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
|
||||||
if (zc->workSpaceSize < neededSpace) {
|
if (zc->workSpaceSize < neededSpace) {
|
||||||
zc->customFree(zc->workSpace);
|
zc->customMem.customFree(zc->customMem.opaque, zc->workSpace);
|
||||||
zc->workSpace = zc->customAlloc(neededSpace);
|
zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace);
|
||||||
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
||||||
zc->workSpaceSize = neededSpace;
|
zc->workSpaceSize = neededSpace;
|
||||||
} }
|
} }
|
||||||
|
|
||||||
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
|
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
|
||||||
zc->hashTable3 = (U32*)(zc->workSpace);
|
XXH64_reset(&zc->xxhState, 0);
|
||||||
zc->hashTable = zc->hashTable3 + h3Size;
|
zc->hashLog3 = hashLog3;
|
||||||
|
zc->hashTable = (U32*)(zc->workSpace);
|
||||||
zc->chainTable = zc->hashTable + hSize;
|
zc->chainTable = zc->hashTable + hSize;
|
||||||
zc->seqStore.buffer = zc->chainTable + chainSize;
|
zc->hashTable3 = zc->chainTable + chainSize;
|
||||||
|
zc->seqStore.buffer = zc->hashTable3 + h3Size;
|
||||||
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
|
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
|
||||||
zc->flagStaticTables = 0;
|
zc->flagStaticTables = 0;
|
||||||
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
|
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
|
||||||
@ -280,6 +303,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
|||||||
zc->lowLimit = 0;
|
zc->lowLimit = 0;
|
||||||
zc->params = params;
|
zc->params = params;
|
||||||
zc->blockSize = blockSize;
|
zc->blockSize = blockSize;
|
||||||
|
zc->frameContentSize = frameContentSize;
|
||||||
|
|
||||||
if (params.cParams.strategy == ZSTD_btopt) {
|
if (params.cParams.strategy == ZSTD_btopt) {
|
||||||
zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer);
|
zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer);
|
||||||
@ -300,6 +324,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
|||||||
zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
|
zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
|
||||||
|
|
||||||
zc->stage = 1;
|
zc->stage = 1;
|
||||||
|
zc->dictID = 0;
|
||||||
zc->loadedDictEnd = 0;
|
zc->loadedDictEnd = 0;
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
@ -314,21 +339,19 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
|||||||
{
|
{
|
||||||
if (srcCCtx->stage!=1) return ERROR(stage_wrong);
|
if (srcCCtx->stage!=1) return ERROR(stage_wrong);
|
||||||
|
|
||||||
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
|
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||||
dstCCtx->customAlloc = srcCCtx->customAlloc;
|
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
|
||||||
dstCCtx->customFree = srcCCtx->customFree;
|
|
||||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
|
|
||||||
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
|
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
|
||||||
|
|
||||||
/* copy tables */
|
/* copy tables */
|
||||||
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||||
const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
|
const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
|
||||||
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
|
const size_t h3Size = (size_t)1 << srcCCtx->hashLog3;
|
||||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||||
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
|
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* copy dictionary pointers */
|
/* copy dictionary offsets */
|
||||||
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
|
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
|
||||||
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
|
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
|
||||||
dstCCtx->nextSrc = srcCCtx->nextSrc;
|
dstCCtx->nextSrc = srcCCtx->nextSrc;
|
||||||
@ -337,6 +360,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
|||||||
dstCCtx->dictLimit = srcCCtx->dictLimit;
|
dstCCtx->dictLimit = srcCCtx->dictLimit;
|
||||||
dstCCtx->lowLimit = srcCCtx->lowLimit;
|
dstCCtx->lowLimit = srcCCtx->lowLimit;
|
||||||
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
|
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
|
||||||
|
dstCCtx->dictID = srcCCtx->dictID;
|
||||||
|
|
||||||
/* copy entropy tables */
|
/* copy entropy tables */
|
||||||
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
|
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
|
||||||
@ -384,8 +408,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
|||||||
/* Frame format description
|
/* Frame format description
|
||||||
Frame Header - [ Block Header - Block ] - Frame End
|
Frame Header - [ Block Header - Block ] - Frame End
|
||||||
1) Frame Header
|
1) Frame Header
|
||||||
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
- 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
||||||
- 1 byte - Frame Descriptor
|
- 1 byte : Frame Header Descriptor
|
||||||
|
- 1-13 bytes : Optional fields
|
||||||
2) Block Header
|
2) Block Header
|
||||||
- 3 bytes, starting with a 2-bits descriptor
|
- 3 bytes, starting with a 2-bits descriptor
|
||||||
Uncompressed, Compressed, Frame End, unused
|
Uncompressed, Compressed, Frame End, unused
|
||||||
@ -398,13 +423,38 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
|||||||
|
|
||||||
/* Frame descriptor
|
/* Frame descriptor
|
||||||
|
|
||||||
1 byte, using :
|
// old
|
||||||
|
1 byte - Alloc :
|
||||||
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||||
bit 4 : minmatch 4(0) or 3(1)
|
bit 4 : reserved for windowLog (must be zero)
|
||||||
bit 5 : reserved (must be zero)
|
bit 5 : reserved (must be zero)
|
||||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||||
|
|
||||||
Optional : content size (0, 1, 2 or 8 bytes)
|
1 byte - checker :
|
||||||
|
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||||
|
bit 2-7 : reserved (must be zero)
|
||||||
|
|
||||||
|
|
||||||
|
// new
|
||||||
|
1 byte - FrameHeaderDescription :
|
||||||
|
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||||
|
bit 2-4 : reserved (must be zero)
|
||||||
|
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
|
||||||
|
bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8)
|
||||||
|
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||||
|
|
||||||
|
Optional : WindowLog (0 or 1 byte)
|
||||||
|
bit 0-2 : octal Fractional (1/8th)
|
||||||
|
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||||
|
|
||||||
|
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||||
|
Automatic adaptation
|
||||||
|
0 : no dictID
|
||||||
|
1 : 1 - 255
|
||||||
|
2 : 256 - 65535
|
||||||
|
4 : all other values
|
||||||
|
|
||||||
|
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||||
0 : unknown
|
0 : unknown
|
||||||
1 : 0-255 bytes
|
1 : 0-255 bytes
|
||||||
2 : 256 - 65535+256
|
2 : 256 - 65535+256
|
||||||
@ -651,7 +701,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
|||||||
size_t u;
|
size_t u;
|
||||||
for (u=0; u<nbSeq; u++) {
|
for (u=0; u<nbSeq; u++) {
|
||||||
U32 const ll = llTable[u];
|
U32 const ll = llTable[u];
|
||||||
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll];
|
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll];
|
||||||
}
|
}
|
||||||
if (seqStorePtr->longLengthID==1)
|
if (seqStorePtr->longLengthID==1)
|
||||||
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
|
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
|
||||||
@ -661,7 +711,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
|||||||
{ const U32* const offsetTable = seqStorePtr->offsetStart;
|
{ const U32* const offsetTable = seqStorePtr->offsetStart;
|
||||||
BYTE* const ofCodeTable = seqStorePtr->offCodeStart;
|
BYTE* const ofCodeTable = seqStorePtr->offCodeStart;
|
||||||
size_t u;
|
size_t u;
|
||||||
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit(offsetTable[u]);
|
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit32(offsetTable[u]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ML codes */
|
/* ML codes */
|
||||||
@ -679,7 +729,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
|||||||
size_t u;
|
size_t u;
|
||||||
for (u=0; u<nbSeq; u++) {
|
for (u=0; u<nbSeq; u++) {
|
||||||
U32 const ml = mlTable[u];
|
U32 const ml = mlTable[u];
|
||||||
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml];
|
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml];
|
||||||
}
|
}
|
||||||
if (seqStorePtr->longLengthID==2)
|
if (seqStorePtr->longLengthID==2)
|
||||||
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
||||||
@ -1427,7 +1477,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
|||||||
if (matchLength > bestLength) {
|
if (matchLength > bestLength) {
|
||||||
if (matchLength > matchEndIdx - matchIndex)
|
if (matchLength > matchEndIdx - matchIndex)
|
||||||
matchEndIdx = matchIndex + (U32)matchLength;
|
matchEndIdx = matchIndex + (U32)matchLength;
|
||||||
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
|
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
|
||||||
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
|
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
|
||||||
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
|
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
|
||||||
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
||||||
@ -1713,14 +1763,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
|||||||
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
||||||
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
||||||
int const gain2 = (int)(mlRep * 3);
|
int const gain2 = (int)(mlRep * 3);
|
||||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
|
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||||
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
|
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
|
||||||
matchLength = mlRep, offset = 0, start = ip;
|
matchLength = mlRep, offset = 0, start = ip;
|
||||||
}
|
}
|
||||||
{ size_t offset2=99999999;
|
{ size_t offset2=99999999;
|
||||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
|
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
|
||||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||||
matchLength = ml2, offset = offset2, start = ip;
|
matchLength = ml2, offset = offset2, start = ip;
|
||||||
continue; /* search a better one */
|
continue; /* search a better one */
|
||||||
@ -1732,14 +1782,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
|||||||
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
||||||
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
||||||
int const gain2 = (int)(ml2 * 4);
|
int const gain2 = (int)(ml2 * 4);
|
||||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
|
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
|
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
|
||||||
matchLength = ml2, offset = 0, start = ip;
|
matchLength = ml2, offset = 0, start = ip;
|
||||||
}
|
}
|
||||||
{ size_t offset2=99999999;
|
{ size_t offset2=99999999;
|
||||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
|
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
|
||||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||||
matchLength = ml2, offset = offset2, start = ip;
|
matchLength = ml2, offset = offset2, start = ip;
|
||||||
continue;
|
continue;
|
||||||
@ -1885,7 +1935,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
|||||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||||
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
||||||
int const gain2 = (int)(repLength * 3);
|
int const gain2 = (int)(repLength * 3);
|
||||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
|
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||||
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
||||||
matchLength = repLength, offset = 0, start = ip;
|
matchLength = repLength, offset = 0, start = ip;
|
||||||
} }
|
} }
|
||||||
@ -1893,8 +1943,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
|||||||
/* search match, depth 1 */
|
/* search match, depth 1 */
|
||||||
{ size_t offset2=99999999;
|
{ size_t offset2=99999999;
|
||||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
|
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
|
||||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||||
matchLength = ml2, offset = offset2, start = ip;
|
matchLength = ml2, offset = offset2, start = ip;
|
||||||
continue; /* search a better one */
|
continue; /* search a better one */
|
||||||
@ -1915,7 +1965,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
|||||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||||
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
||||||
int gain2 = (int)(repLength * 4);
|
int gain2 = (int)(repLength * 4);
|
||||||
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
|
int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||||
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
||||||
matchLength = repLength, offset = 0, start = ip;
|
matchLength = repLength, offset = 0, start = ip;
|
||||||
} }
|
} }
|
||||||
@ -1923,8 +1973,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
|||||||
/* search match, depth 2 */
|
/* search match, depth 2 */
|
||||||
{ size_t offset2=99999999;
|
{ size_t offset2=99999999;
|
||||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
|
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
|
||||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||||
matchLength = ml2, offset = offset2, start = ip;
|
matchLength = ml2, offset = offset2, start = ip;
|
||||||
continue;
|
continue;
|
||||||
@ -1934,7 +1984,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
|||||||
|
|
||||||
/* catch up */
|
/* catch up */
|
||||||
if (offset) {
|
if (offset) {
|
||||||
U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
||||||
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
||||||
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
||||||
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
||||||
@ -2039,34 +2089,37 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||||
void* dst, size_t dstCapacity,
|
void* dst, size_t dstCapacity,
|
||||||
const void* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
size_t blockSize = zc->blockSize;
|
size_t blockSize = cctx->blockSize;
|
||||||
size_t remaining = srcSize;
|
size_t remaining = srcSize;
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
BYTE* const ostart = (BYTE*)dst;
|
BYTE* const ostart = (BYTE*)dst;
|
||||||
BYTE* op = ostart;
|
BYTE* op = ostart;
|
||||||
const U32 maxDist = 1 << zc->params.cParams.windowLog;
|
const U32 maxDist = 1 << cctx->params.cParams.windowLog;
|
||||||
ZSTD_stats_t* stats = &zc->seqStore.stats;
|
ZSTD_stats_t* stats = &cctx->seqStore.stats;
|
||||||
ZSTD_statsInit(stats);
|
ZSTD_statsInit(stats);
|
||||||
|
|
||||||
|
if (cctx->params.fParams.checksumFlag)
|
||||||
|
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||||
|
|
||||||
while (remaining) {
|
while (remaining) {
|
||||||
size_t cSize;
|
size_t cSize;
|
||||||
ZSTD_statsResetFreqs(stats);
|
ZSTD_statsResetFreqs(stats); /* debug only */
|
||||||
|
|
||||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||||
if (remaining < blockSize) blockSize = remaining;
|
if (remaining < blockSize) blockSize = remaining;
|
||||||
|
|
||||||
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
|
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
|
||||||
/* enforce maxDist */
|
/* enforce maxDist */
|
||||||
U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
|
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
|
||||||
if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit;
|
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
|
||||||
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
|
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
|
||||||
}
|
}
|
||||||
|
|
||||||
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
||||||
if (ZSTD_isError(cSize)) return cSize;
|
if (ZSTD_isError(cSize)) return cSize;
|
||||||
|
|
||||||
if (cSize == 0) { /* block is not compressible */
|
if (cSize == 0) { /* block is not compressible */
|
||||||
@ -2086,33 +2139,47 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
|||||||
op += cSize;
|
op += cSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
|
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
|
||||||
return op-ostart;
|
return op-ostart;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
|
||||||
{ BYTE* const op = (BYTE*)dst;
|
{ BYTE* const op = (BYTE*)dst;
|
||||||
U32 const fcsId = params.fParams.contentSizeFlag ?
|
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||||
(pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */
|
U32 const checksumFlag = params.fParams.checksumFlag;
|
||||||
|
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||||
|
U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
|
||||||
|
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||||
|
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||||
|
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
|
||||||
0;
|
0;
|
||||||
BYTE const fdescriptor = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
|
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) );
|
||||||
| (fcsId << 6) );
|
size_t pos;
|
||||||
size_t const hSize = ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
|
|
||||||
if (hSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||||
|
|
||||||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||||
op[4] = fdescriptor;
|
op[4] = frameHeaderDecriptionByte; pos=5;
|
||||||
switch(fcsId)
|
if (!directModeFlag) op[pos++] = windowLogByte;
|
||||||
|
switch(dictIDSizeCode)
|
||||||
{
|
{
|
||||||
default: /* impossible */
|
default: /* impossible */
|
||||||
case 0 : break;
|
case 0 : break;
|
||||||
case 1 : op[5] = (BYTE)(pledgedSrcSize); break;
|
case 1 : op[pos] = (BYTE)(dictID); pos++; break;
|
||||||
case 2 : MEM_writeLE16(op+5, (U16)(pledgedSrcSize-256)); break;
|
case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break;
|
||||||
case 3 : MEM_writeLE64(op+5, (U64)(pledgedSrcSize)); break;
|
case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
|
||||||
}
|
}
|
||||||
return hSize;
|
switch(fcsCode)
|
||||||
|
{
|
||||||
|
default: /* impossible */
|
||||||
|
case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break;
|
||||||
|
case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
|
||||||
|
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
|
||||||
|
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
|
||||||
|
}
|
||||||
|
return pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -2126,7 +2193,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
|
|||||||
|
|
||||||
if (zc->stage==0) return ERROR(stage_wrong);
|
if (zc->stage==0) return ERROR(stage_wrong);
|
||||||
if (frame && (zc->stage==1)) { /* copy saved header */
|
if (frame && (zc->stage==1)) { /* copy saved header */
|
||||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize);
|
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
|
||||||
if (ZSTD_isError(fhSize)) return fhSize;
|
if (ZSTD_isError(fhSize)) return fhSize;
|
||||||
dstCapacity -= fhSize;
|
dstCapacity -= fhSize;
|
||||||
dst = (char*)dst + fhSize;
|
dst = (char*)dst + fhSize;
|
||||||
@ -2244,53 +2311,59 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
|||||||
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
/* note : magic number already checked */
|
/* note : magic number already checked */
|
||||||
size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
|
size_t const dictSizeStart = dictSize;
|
||||||
short offcodeNCount[MaxOff+1];
|
|
||||||
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
|
||||||
short matchlengthNCount[MaxML+1];
|
|
||||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
|
||||||
short litlengthNCount[MaxLL+1];
|
|
||||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
|
||||||
|
|
||||||
size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
|
{ size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
|
||||||
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
zc->flagStaticTables = 1;
|
zc->flagStaticTables = 1;
|
||||||
dict = (const char*)dict + hufHeaderSize;
|
dict = (const char*)dict + hufHeaderSize;
|
||||||
dictSize -= hufHeaderSize;
|
dictSize -= hufHeaderSize;
|
||||||
|
}
|
||||||
|
|
||||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
{ short offcodeNCount[MaxOff+1];
|
||||||
|
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
||||||
|
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
{ size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
dict = (const char*)dict + offcodeHeaderSize;
|
dict = (const char*)dict + offcodeHeaderSize;
|
||||||
dictSize -= offcodeHeaderSize;
|
dictSize -= offcodeHeaderSize;
|
||||||
|
}
|
||||||
|
|
||||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
{ short matchlengthNCount[MaxML+1];
|
||||||
|
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||||
|
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
{ size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
dict = (const char*)dict + matchlengthHeaderSize;
|
dict = (const char*)dict + matchlengthHeaderSize;
|
||||||
dictSize -= matchlengthHeaderSize;
|
dictSize -= matchlengthHeaderSize;
|
||||||
|
}
|
||||||
|
|
||||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
{ short litlengthNCount[MaxLL+1];
|
||||||
|
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||||
|
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
{ size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
|
dictSize -= litlengthHeaderSize;
|
||||||
|
}
|
||||||
|
|
||||||
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
return (dictSizeStart-dictSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** ZSTD_compress_insertDictionary() :
|
/** ZSTD_compress_insertDictionary() :
|
||||||
* @return : 0, or an error code */
|
* @return : 0, or an error code */
|
||||||
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
if ((dict==NULL) || (dictSize<=4)) return 0;
|
if ((dict==NULL) || (dictSize<=8)) return 0;
|
||||||
|
|
||||||
/* default : dict is pure content */
|
/* default : dict is pure content */
|
||||||
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
|
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
|
||||||
|
zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
|
||||||
|
|
||||||
/* known magic number : dict is parsed for entropy stats and content */
|
/* known magic number : dict is parsed for entropy stats and content */
|
||||||
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4;
|
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8;
|
||||||
if (ZSTD_isError(eSize)) return eSize;
|
if (ZSTD_isError(eSize)) return eSize;
|
||||||
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
|
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
|
||||||
}
|
}
|
||||||
@ -2303,11 +2376,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
|||||||
const void* dict, size_t dictSize,
|
const void* dict, size_t dictSize,
|
||||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||||
{
|
{
|
||||||
{ U32 const hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
|
size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
|
||||||
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; }
|
if (ZSTD_isError(resetError)) return resetError;
|
||||||
|
|
||||||
{ size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1);
|
|
||||||
if (ZSTD_isError(resetError)) return resetError; }
|
|
||||||
|
|
||||||
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
|
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
|
||||||
}
|
}
|
||||||
@ -2330,9 +2400,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
|
|||||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
|
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||||
params.fParams.contentSizeFlag = 0;
|
|
||||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
|
||||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
||||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
||||||
}
|
}
|
||||||
@ -2348,30 +2417,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
|||||||
/*! ZSTD_compressEnd() :
|
/*! ZSTD_compressEnd() :
|
||||||
* Write frame epilogue.
|
* Write frame epilogue.
|
||||||
* @return : nb of bytes written into dst (or an error code) */
|
* @return : nb of bytes written into dst (or an error code) */
|
||||||
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
|
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
|
||||||
{
|
{
|
||||||
BYTE* op = (BYTE*)dst;
|
BYTE* op = (BYTE*)dst;
|
||||||
size_t fhSize = 0;
|
size_t fhSize = 0;
|
||||||
|
|
||||||
/* not even init ! */
|
/* not even init ! */
|
||||||
if (zc->stage==0) return ERROR(stage_wrong);
|
if (cctx->stage==0) return ERROR(stage_wrong);
|
||||||
|
|
||||||
/* special case : empty frame */
|
/* special case : empty frame */
|
||||||
if (zc->stage==1) {
|
if (cctx->stage==1) {
|
||||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0);
|
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
|
||||||
if (ZSTD_isError(fhSize)) return fhSize;
|
if (ZSTD_isError(fhSize)) return fhSize;
|
||||||
dstCapacity -= fhSize;
|
dstCapacity -= fhSize;
|
||||||
op += fhSize;
|
op += fhSize;
|
||||||
zc->stage = 2;
|
cctx->stage = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* frame epilogue */
|
/* frame epilogue */
|
||||||
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
|
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
|
||||||
op[0] = (BYTE)(bt_end << 6);
|
{ U32 const checksum = cctx->params.fParams.checksumFlag ?
|
||||||
op[1] = 0;
|
(U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
|
||||||
op[2] = 0;
|
0;
|
||||||
|
op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
|
||||||
|
op[1] = (BYTE)(checksum>>8);
|
||||||
|
op[2] = (BYTE)checksum;
|
||||||
|
}
|
||||||
|
|
||||||
zc->stage = 0; /* return to "created by not init" status */
|
cctx->stage = 0; /* return to "created but not init" status */
|
||||||
return 3+fhSize;
|
return 3+fhSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2434,10 +2507,10 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
|||||||
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
|
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
|
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
ZSTD_adjustCParams(¶ms.cParams, srcSize, dictSize);
|
|
||||||
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
|
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2452,10 +2525,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
|
|||||||
size_t result;
|
size_t result;
|
||||||
ZSTD_CCtx ctxBody;
|
ZSTD_CCtx ctxBody;
|
||||||
memset(&ctxBody, 0, sizeof(ctxBody));
|
memset(&ctxBody, 0, sizeof(ctxBody));
|
||||||
ctxBody.customAlloc = malloc;
|
memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||||
ctxBody.customFree = free;
|
|
||||||
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
|
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
|
||||||
ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
|
ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2590,5 +2662,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
|
|||||||
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
|
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
|
||||||
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
|
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
|
||||||
}
|
}
|
||||||
|
cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
|
||||||
return cp;
|
return cp;
|
||||||
}
|
}
|
||||||
|
@ -41,10 +41,10 @@
|
|||||||
***************************************/
|
***************************************/
|
||||||
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
|
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
|
||||||
{
|
{
|
||||||
ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1);
|
ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1);
|
||||||
ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1);
|
ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1);
|
||||||
ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1);
|
ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1);
|
||||||
ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1);
|
ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1);
|
||||||
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
|
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
|||||||
U32 price, u;
|
U32 price, u;
|
||||||
|
|
||||||
if (litLength == 0)
|
if (litLength == 0)
|
||||||
return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1);
|
return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
|
||||||
|
|
||||||
/* literals */
|
/* literals */
|
||||||
if (ssPtr->cachedLiterals == literals) {
|
if (ssPtr->cachedLiterals == literals) {
|
||||||
@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
|||||||
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
|
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
|
||||||
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
|
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
|
||||||
for (u=0; u < additional; u++)
|
for (u=0; u < additional; u++)
|
||||||
price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1);
|
price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1);
|
||||||
ssPtr->cachedPrice = price;
|
ssPtr->cachedPrice = price;
|
||||||
ssPtr->cachedLitLength = litLength;
|
ssPtr->cachedLitLength = litLength;
|
||||||
} else {
|
} else {
|
||||||
price = litLength * ssPtr->log2litSum;
|
price = litLength * ssPtr->log2litSum;
|
||||||
for (u=0; u < litLength; u++)
|
for (u=0; u < litLength; u++)
|
||||||
price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1);
|
price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1);
|
||||||
|
|
||||||
if (litLength >= 12) {
|
if (litLength >= 12) {
|
||||||
ssPtr->cachedLiterals = literals;
|
ssPtr->cachedLiterals = literals;
|
||||||
@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
|||||||
24, 24, 24, 24, 24, 24, 24, 24,
|
24, 24, 24, 24, 24, 24, 24, 24,
|
||||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||||
const BYTE LL_deltaCode = 19;
|
const BYTE LL_deltaCode = 19;
|
||||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||||
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1);
|
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return price;
|
return price;
|
||||||
@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
|||||||
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
|
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
|
||||||
{
|
{
|
||||||
/* offset */
|
/* offset */
|
||||||
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
|
||||||
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
|
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
|
||||||
|
|
||||||
/* match Length */
|
/* match Length */
|
||||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||||
@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
|
|||||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||||
const BYTE ML_deltaCode = 36;
|
const BYTE ML_deltaCode = 36;
|
||||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||||
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
|
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
|
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
|
||||||
@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
|||||||
24, 24, 24, 24, 24, 24, 24, 24,
|
24, 24, 24, 24, 24, 24, 24, 24,
|
||||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||||
const BYTE LL_deltaCode = 19;
|
const BYTE LL_deltaCode = 19;
|
||||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||||
seqStorePtr->litLengthFreq[llCode]++;
|
seqStorePtr->litLengthFreq[llCode]++;
|
||||||
seqStorePtr->litLengthSum++;
|
seqStorePtr->litLengthSum++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* match offset */
|
/* match offset */
|
||||||
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
{ BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
|
||||||
seqStorePtr->offCodeSum++;
|
seqStorePtr->offCodeSum++;
|
||||||
seqStorePtr->offCodeFreq[offCode]++;
|
seqStorePtr->offCodeFreq[offCode]++;
|
||||||
}
|
}
|
||||||
@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
|||||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||||
const BYTE ML_deltaCode = 36;
|
const BYTE ML_deltaCode = 36;
|
||||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||||
seqStorePtr->matchLengthFreq[mlCode]++;
|
seqStorePtr->matchLengthFreq[mlCode]++;
|
||||||
seqStorePtr->matchLengthSum++;
|
seqStorePtr->matchLengthSum++;
|
||||||
}
|
}
|
||||||
|
@ -59,12 +59,12 @@
|
|||||||
/* **************************************************************
|
/* **************************************************************
|
||||||
* Includes
|
* Includes
|
||||||
****************************************************************/
|
****************************************************************/
|
||||||
#include <stdlib.h> /* malloc, free, qsort */
|
|
||||||
#include <string.h> /* memcpy, memset */
|
#include <string.h> /* memcpy, memset */
|
||||||
#include <stdio.h> /* printf (debug) */
|
#include <stdio.h> /* printf (debug) */
|
||||||
#include "huf_static.h"
|
|
||||||
#include "bitstream.h"
|
#include "bitstream.h"
|
||||||
#include "fse.h" /* header compression */
|
#include "fse.h" /* header compression */
|
||||||
|
#define HUF_STATIC_LINKING_ONLY
|
||||||
|
#include "huf.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -35,9 +35,9 @@
|
|||||||
***************************************/
|
***************************************/
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
|
||||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
#define ZBUFF_STATIC_LINKING_ONLY
|
||||||
#include "zbuff_static.h"
|
#include "zbuff.h"
|
||||||
|
|
||||||
|
|
||||||
/*-***************************************************************************
|
/*-***************************************************************************
|
||||||
@ -82,15 +82,13 @@ struct ZBUFF_DCtx_s {
|
|||||||
size_t blockSize;
|
size_t blockSize;
|
||||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||||
size_t lhSize;
|
size_t lhSize;
|
||||||
ZSTD_allocFunction customAlloc;
|
ZSTD_customMem customMem;
|
||||||
ZSTD_freeFunction customFree;
|
|
||||||
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
|
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
|
||||||
|
|
||||||
|
|
||||||
ZBUFF_DCtx* ZBUFF_createDCtx(void)
|
ZBUFF_DCtx* ZBUFF_createDCtx(void)
|
||||||
{
|
{
|
||||||
ZSTD_customMem customMem = { NULL, NULL };
|
return ZBUFF_createDCtx_advanced(defaultCustomMem);
|
||||||
return ZBUFF_createDCtx_advanced(customMem);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
|
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
|
||||||
@ -98,25 +96,17 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
|
|||||||
ZBUFF_DCtx* zbd;
|
ZBUFF_DCtx* zbd;
|
||||||
|
|
||||||
if (!customMem.customAlloc && !customMem.customFree)
|
if (!customMem.customAlloc && !customMem.customFree)
|
||||||
{
|
customMem = defaultCustomMem;
|
||||||
zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx));
|
|
||||||
if (zbd==NULL) return NULL;
|
|
||||||
zbd->customAlloc = malloc;
|
|
||||||
zbd->customFree = free;
|
|
||||||
zbd->zd = ZSTD_createDCtx();
|
|
||||||
zbd->stage = ZBUFFds_init;
|
|
||||||
return zbd;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!customMem.customAlloc || !customMem.customFree)
|
if (!customMem.customAlloc || !customMem.customFree)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx));
|
zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
|
||||||
if (zbd==NULL) return NULL;
|
if (zbd==NULL) return NULL;
|
||||||
memset(zbd, 0, sizeof(ZBUFF_DCtx));
|
memset(zbd, 0, sizeof(ZBUFF_DCtx));
|
||||||
zbd->customAlloc = customMem.customAlloc;
|
memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||||
zbd->customFree = customMem.customFree;
|
|
||||||
zbd->zd = ZSTD_createDCtx_advanced(customMem);
|
zbd->zd = ZSTD_createDCtx_advanced(customMem);
|
||||||
|
if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
|
||||||
zbd->stage = ZBUFFds_init;
|
zbd->stage = ZBUFFds_init;
|
||||||
return zbd;
|
return zbd;
|
||||||
}
|
}
|
||||||
@ -125,9 +115,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
|
|||||||
{
|
{
|
||||||
if (zbd==NULL) return 0; /* support free on null */
|
if (zbd==NULL) return 0; /* support free on null */
|
||||||
ZSTD_freeDCtx(zbd->zd);
|
ZSTD_freeDCtx(zbd->zd);
|
||||||
zbd->customFree(zbd->inBuff);
|
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||||
zbd->customFree(zbd->outBuff);
|
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||||
zbd->customFree(zbd);
|
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -147,6 +137,15 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* internal util function */
|
||||||
|
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
|
{
|
||||||
|
size_t const length = MIN(dstCapacity, srcSize);
|
||||||
|
memcpy(dst, src, length);
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* *** Decompression *** */
|
/* *** Decompression *** */
|
||||||
|
|
||||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||||
@ -192,20 +191,22 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
|||||||
if (ZSTD_isError(h2Result)) return h2Result;
|
if (ZSTD_isError(h2Result)) return h2Result;
|
||||||
} }
|
} }
|
||||||
|
|
||||||
|
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||||
|
|
||||||
/* Frame header instruct buffer sizes */
|
/* Frame header instruct buffer sizes */
|
||||||
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||||
zbd->blockSize = blockSize;
|
zbd->blockSize = blockSize;
|
||||||
if (zbd->inBuffSize < blockSize) {
|
if (zbd->inBuffSize < blockSize) {
|
||||||
zbd->customFree(zbd->inBuff);
|
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||||
zbd->inBuffSize = blockSize;
|
zbd->inBuffSize = blockSize;
|
||||||
zbd->inBuff = (char*)zbd->customAlloc(blockSize);
|
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||||
}
|
}
|
||||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||||
if (zbd->outBuffSize < neededOutSize) {
|
if (zbd->outBuffSize < neededOutSize) {
|
||||||
zbd->customFree(zbd->outBuff);
|
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||||
zbd->outBuffSize = neededOutSize;
|
zbd->outBuffSize = neededOutSize;
|
||||||
zbd->outBuff = (char*)zbd->customAlloc(neededOutSize);
|
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
|
||||||
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
||||||
} } }
|
} } }
|
||||||
zbd->stage = ZBUFFds_read;
|
zbd->stage = ZBUFFds_read;
|
||||||
@ -218,12 +219,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||||
|
const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
|
||||||
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
|
||||||
ip, neededInSize);
|
ip, neededInSize);
|
||||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||||
ip += neededInSize;
|
ip += neededInSize;
|
||||||
if (!decodedSize) break; /* this was just a header */
|
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
||||||
zbd->outEnd = zbd->outStart + decodedSize;
|
zbd->outEnd = zbd->outStart + decodedSize;
|
||||||
zbd->stage = ZBUFFds_flush;
|
zbd->stage = ZBUFFds_flush;
|
||||||
break;
|
break;
|
||||||
@ -243,12 +245,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
|||||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||||
|
|
||||||
/* decode loaded input */
|
/* decode loaded input */
|
||||||
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
{ const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
|
||||||
|
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||||
zbd->inBuff, neededInSize);
|
zbd->inBuff, neededInSize);
|
||||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||||
zbd->inPos = 0; /* input is consumed */
|
zbd->inPos = 0; /* input is consumed */
|
||||||
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||||
zbd->outEnd = zbd->outStart + decodedSize;
|
zbd->outEnd = zbd->outStart + decodedSize;
|
||||||
zbd->stage = ZBUFFds_flush;
|
zbd->stage = ZBUFFds_flush;
|
||||||
// break; /* ZBUFFds_flush follows */
|
// break; /* ZBUFFds_flush follows */
|
||||||
@ -276,7 +279,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
|||||||
*srcSizePtr = ip-istart;
|
*srcSizePtr = ip-istart;
|
||||||
*dstCapacityPtr = op-ostart;
|
*dstCapacityPtr = op-ostart;
|
||||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
// if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||||
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
||||||
return nextSrcSizeHint;
|
return nextSrcSizeHint;
|
||||||
}
|
}
|
||||||
|
@ -53,15 +53,18 @@
|
|||||||
/*-*******************************************************
|
/*-*******************************************************
|
||||||
* Dependencies
|
* Dependencies
|
||||||
*********************************************************/
|
*********************************************************/
|
||||||
#include <stdlib.h> /* calloc */
|
|
||||||
#include <string.h> /* memcpy, memmove */
|
#include <string.h> /* memcpy, memmove */
|
||||||
#include <stdio.h> /* debug only : printf */
|
#include <stdio.h> /* debug only : printf */
|
||||||
#include "mem.h" /* low level memory routines */
|
#include "mem.h" /* low level memory routines */
|
||||||
|
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||||
|
#include "xxhash.h" /* XXH64_* */
|
||||||
|
#define FSE_STATIC_LINKING_ONLY
|
||||||
|
#include "fse.h"
|
||||||
|
#define HUF_STATIC_LINKING_ONLY
|
||||||
|
#include "huf.h"
|
||||||
#include "zstd_internal.h"
|
#include "zstd_internal.h"
|
||||||
#include "fse_static.h"
|
|
||||||
#include "huf_static.h"
|
|
||||||
|
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||||
# include "zstd_legacy.h"
|
# include "zstd_legacy.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -101,7 +104,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
|||||||
* Context management
|
* Context management
|
||||||
***************************************************************/
|
***************************************************************/
|
||||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
|
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
||||||
|
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
|
||||||
|
|
||||||
struct ZSTD_DCtx_s
|
struct ZSTD_DCtx_s
|
||||||
{
|
{
|
||||||
@ -116,10 +120,11 @@ struct ZSTD_DCtx_s
|
|||||||
size_t expected;
|
size_t expected;
|
||||||
size_t headerSize;
|
size_t headerSize;
|
||||||
ZSTD_frameParams fParams;
|
ZSTD_frameParams fParams;
|
||||||
ZSTD_allocFunction customAlloc;
|
XXH64_state_t xxhState;
|
||||||
ZSTD_freeFunction customFree;
|
ZSTD_customMem customMem;
|
||||||
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||||
ZSTD_dStage stage;
|
ZSTD_dStage stage;
|
||||||
|
U32 dictID;
|
||||||
U32 flagRepeatTable;
|
U32 flagRepeatTable;
|
||||||
const BYTE* litPtr;
|
const BYTE* litPtr;
|
||||||
size_t litBufSize;
|
size_t litBufSize;
|
||||||
@ -140,46 +145,37 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
|||||||
dctx->dictEnd = NULL;
|
dctx->dictEnd = NULL;
|
||||||
dctx->hufTableX4[0] = HufLog;
|
dctx->hufTableX4[0] = HufLog;
|
||||||
dctx->flagRepeatTable = 0;
|
dctx->flagRepeatTable = 0;
|
||||||
|
dctx->dictID = 0;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_DCtx* ZSTD_createDCtx(void)
|
|
||||||
{
|
|
||||||
ZSTD_customMem customMem = { NULL, NULL };
|
|
||||||
return ZSTD_createDCtx_advanced(customMem);
|
|
||||||
}
|
|
||||||
|
|
||||||
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
||||||
{
|
{
|
||||||
ZSTD_DCtx* dctx;
|
ZSTD_DCtx* dctx;
|
||||||
|
|
||||||
if (!customMem.customAlloc && !customMem.customFree)
|
if (!customMem.customAlloc && !customMem.customFree)
|
||||||
{
|
customMem = defaultCustomMem;
|
||||||
dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx));
|
|
||||||
if (!dctx) return NULL;
|
|
||||||
dctx->customAlloc = malloc;
|
|
||||||
dctx->customFree = free;
|
|
||||||
|
|
||||||
ZSTD_decompressBegin(dctx);
|
|
||||||
return dctx;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!customMem.customAlloc || !customMem.customFree)
|
if (!customMem.customAlloc || !customMem.customFree)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx));
|
dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
|
||||||
if (!dctx) return NULL;
|
if (!dctx) return NULL;
|
||||||
dctx->customAlloc = customMem.customAlloc;
|
memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||||
dctx->customFree = customMem.customFree;
|
|
||||||
|
|
||||||
ZSTD_decompressBegin(dctx);
|
ZSTD_decompressBegin(dctx);
|
||||||
return dctx;
|
return dctx;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ZSTD_DCtx* ZSTD_createDCtx(void)
|
||||||
|
{
|
||||||
|
return ZSTD_createDCtx_advanced(defaultCustomMem);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
||||||
{
|
{
|
||||||
dctx->customFree(dctx);
|
if (dctx==NULL) return 0; /* support free on NULL */
|
||||||
|
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
|
||||||
return 0; /* reserved as a potential error code in the future */
|
return 0; /* reserved as a potential error code in the future */
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -211,17 +207,42 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
|||||||
|
|
||||||
/* Frame descriptor
|
/* Frame descriptor
|
||||||
|
|
||||||
1 byte, using :
|
// old
|
||||||
|
1 byte - Alloc :
|
||||||
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||||
bit 4 : minmatch 4(0) or 3(1)
|
bit 4 : reserved for windowLog (must be zero)
|
||||||
bit 5 : reserved (must be zero)
|
bit 5 : reserved (must be zero)
|
||||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||||
|
|
||||||
Optional : content size (0, 1, 2 or 8 bytes)
|
1 byte - checker :
|
||||||
0 : unknown
|
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||||
1 : 0-255 bytes
|
bit 2-7 : reserved (must be zero)
|
||||||
2 : 256 - 65535+256
|
|
||||||
8 : up to 16 exa
|
// new
|
||||||
|
1 byte - FrameHeaderDescription :
|
||||||
|
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||||
|
bit 2-4 : reserved (must be zero)
|
||||||
|
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
|
||||||
|
bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
|
||||||
|
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||||
|
|
||||||
|
Optional : WindowLog (0 or 1 byte)
|
||||||
|
bit 0-2 : octal Fractional (1/8th)
|
||||||
|
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||||
|
|
||||||
|
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||||
|
Automatic adaptation
|
||||||
|
0 : no dictID
|
||||||
|
1 : 1 - 255
|
||||||
|
2 : 256 - 65535
|
||||||
|
4 : all other values
|
||||||
|
|
||||||
|
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||||
|
0 : unknown (fcfs==0 and swl==0)
|
||||||
|
1 : 0-255 bytes (fcfs==0 and swl==1)
|
||||||
|
2 : 256 - 65535+256 (fcfs==1)
|
||||||
|
4 : 0 - 4GB-1 (fcfs==2)
|
||||||
|
8 : 0 - 16EB-1 (fcfs==3)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
||||||
@ -297,13 +318,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
|||||||
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||||
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
|
{ BYTE const fhd = ((const BYTE*)src)[4];
|
||||||
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; }
|
U32 const dictID= fhd & 3;
|
||||||
|
U32 const directMode = (fhd >> 5) & 1;
|
||||||
|
U32 const fcsId = fhd >> 6;
|
||||||
|
return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
|
||||||
|
+ (directMode && !ZSTD_fcs_fieldSize[fcsId]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/** ZSTD_getFrameParams() :
|
/** ZSTD_getFrameParams() :
|
||||||
* decode Frame Header, or provide expected `srcSize`.
|
* decode Frame Header, or require larger `srcSize`.
|
||||||
* @return : 0, `fparamsPtr` is correctly filled,
|
* @return : 0, `fparamsPtr` is correctly filled,
|
||||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
* >0, `srcSize` is too small, result is expected `srcSize`,
|
||||||
* or an error code, which can be tested using ZSTD_isError() */
|
* or an error code, which can be tested using ZSTD_isError() */
|
||||||
@ -312,24 +338,63 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
|
|
||||||
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
|
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
|
||||||
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
|
||||||
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
|
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
|
||||||
|
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||||
|
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||||
|
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return ERROR(prefix_unknown);
|
||||||
|
}
|
||||||
|
|
||||||
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
||||||
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
||||||
if (srcSize < fhsize) return fhsize; }
|
if (srcSize < fhsize) return fhsize; }
|
||||||
|
|
||||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
{ BYTE const fhdByte = ip[4];
|
||||||
{ BYTE const frameDesc = ip[4];
|
size_t pos = 5;
|
||||||
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
U32 const dictIDSizeCode = fhdByte&3;
|
||||||
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
|
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||||
switch(frameDesc >> 6) /* fcsId */
|
U32 const directMode = (fhdByte>>5)&1;
|
||||||
|
U32 const fcsID = fhdByte>>6;
|
||||||
|
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
|
||||||
|
U32 windowSize = 0;
|
||||||
|
U32 dictID = 0;
|
||||||
|
U64 frameContentSize = 0;
|
||||||
|
if ((fhdByte & 0x18) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
|
||||||
|
if (!directMode) {
|
||||||
|
BYTE const wlByte = ip[pos++];
|
||||||
|
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||||
|
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
||||||
|
windowSize = (1U << windowLog);
|
||||||
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||||
|
}
|
||||||
|
|
||||||
|
switch(dictIDSizeCode)
|
||||||
{
|
{
|
||||||
default: /* impossible */
|
default: /* impossible */
|
||||||
case 0 : fparamsPtr->frameContentSize = 0; break;
|
case 0 : break;
|
||||||
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
|
case 1 : dictID = ip[pos]; pos++; break;
|
||||||
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break;
|
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||||
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
|
case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||||
} }
|
}
|
||||||
|
switch(fcsID)
|
||||||
|
{
|
||||||
|
default: /* impossible */
|
||||||
|
case 0 : if (directMode) frameContentSize = ip[pos]; break;
|
||||||
|
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||||
|
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
||||||
|
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||||
|
}
|
||||||
|
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||||
|
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
||||||
|
fparamsPtr->frameContentSize = frameContentSize;
|
||||||
|
fparamsPtr->windowSize = windowSize;
|
||||||
|
fparamsPtr->dictID = dictID;
|
||||||
|
fparamsPtr->checksumFlag = checksumFlag;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -337,10 +402,11 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|||||||
/** ZSTD_decodeFrameHeader() :
|
/** ZSTD_decodeFrameHeader() :
|
||||||
* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
|
* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
|
||||||
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
|
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
|
||||||
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
|
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
|
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||||
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||||
|
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -611,9 +677,10 @@ typedef struct {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
||||||
{
|
{
|
||||||
/* Literal length */
|
seq_t seq;
|
||||||
|
|
||||||
U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
|
U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
|
||||||
U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
|
U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
|
||||||
U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||||
@ -669,13 +736,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|||||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||||
seqState->prevOffset[0] = offset;
|
seqState->prevOffset[0] = offset;
|
||||||
}
|
}
|
||||||
seq->offset = offset;
|
seq.offset = offset;
|
||||||
}
|
}
|
||||||
|
|
||||||
seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||||
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
|
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
|
||||||
|
|
||||||
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||||
if (MEM_32bits() ||
|
if (MEM_32bits() ||
|
||||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
|
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
|
||||||
|
|
||||||
@ -684,6 +751,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
|||||||
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||||
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||||
|
|
||||||
|
return seq;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -789,11 +858,7 @@ static size_t ZSTD_decompressSequences(
|
|||||||
|
|
||||||
/* Regen sequences */
|
/* Regen sequences */
|
||||||
if (nbSeq) {
|
if (nbSeq) {
|
||||||
seq_t sequence;
|
|
||||||
seqState_t seqState;
|
seqState_t seqState;
|
||||||
|
|
||||||
memset(&sequence, 0, sizeof(sequence));
|
|
||||||
sequence.offset = REPCODE_STARTVALUE;
|
|
||||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
|
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
|
||||||
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
||||||
@ -803,7 +868,7 @@ static size_t ZSTD_decompressSequences(
|
|||||||
|
|
||||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
|
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
|
||||||
nbSeq--;
|
nbSeq--;
|
||||||
ZSTD_decodeSequence(&sequence, &seqState);
|
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
|
||||||
|
|
||||||
#if 0 /* debug */
|
#if 0 /* debug */
|
||||||
static BYTE* start = NULL;
|
static BYTE* start = NULL;
|
||||||
@ -814,7 +879,7 @@ static size_t ZSTD_decompressSequences(
|
|||||||
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
|
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||||
op += oneSeqSize;
|
op += oneSeqSize;
|
||||||
} }
|
} }
|
||||||
@ -882,10 +947,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
const BYTE* const iend = ip + srcSize;
|
const BYTE* const iend = ip + srcSize;
|
||||||
BYTE* const ostart = (BYTE* const)dst;
|
BYTE* const ostart = (BYTE* const)dst;
|
||||||
BYTE* op = ostart;
|
|
||||||
BYTE* const oend = ostart + dstCapacity;
|
BYTE* const oend = ostart + dstCapacity;
|
||||||
|
BYTE* op = ostart;
|
||||||
size_t remainingSize = srcSize;
|
size_t remainingSize = srcSize;
|
||||||
blockProperties_t blockProperties = { bt_compressed, 0 };
|
|
||||||
|
|
||||||
/* check */
|
/* check */
|
||||||
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||||
@ -901,6 +965,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||||||
/* Loop on each block */
|
/* Loop on each block */
|
||||||
while (1) {
|
while (1) {
|
||||||
size_t decodedSize=0;
|
size_t decodedSize=0;
|
||||||
|
blockProperties_t blockProperties;
|
||||||
size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||||
|
|
||||||
@ -929,6 +994,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||||||
if (cBlockSize == 0) break; /* bt_end */
|
if (cBlockSize == 0) break; /* bt_end */
|
||||||
|
|
||||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||||
|
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||||
op += decodedSize;
|
op += decodedSize;
|
||||||
ip += cBlockSize;
|
ip += cBlockSize;
|
||||||
remainingSize -= cBlockSize;
|
remainingSize -= cBlockSize;
|
||||||
@ -954,7 +1020,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|||||||
const void* dict, size_t dictSize)
|
const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||||
{ const U32 magicNumber = MEM_readLE32(src);
|
{ U32 const magicNumber = MEM_readLE32(src);
|
||||||
if (ZSTD_isLegacy(magicNumber))
|
if (ZSTD_isLegacy(magicNumber))
|
||||||
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
|
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
|
||||||
}
|
}
|
||||||
@ -975,7 +1041,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
|||||||
{
|
{
|
||||||
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
|
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
|
||||||
size_t regenSize;
|
size_t regenSize;
|
||||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
if (dctx==NULL) return ERROR(memory_allocation);
|
if (dctx==NULL) return ERROR(memory_allocation);
|
||||||
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||||
ZSTD_freeDCtx(dctx);
|
ZSTD_freeDCtx(dctx);
|
||||||
@ -995,17 +1061,30 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
|
|||||||
return dctx->expected;
|
return dctx->expected;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
|
||||||
|
{
|
||||||
|
return dctx->stage == ZSTDds_skipFrame;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ZSTD_decompressContinue() :
|
||||||
|
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
||||||
|
* or an error code, which can be tested using ZSTD_isError() */
|
||||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
/* Sanity check */
|
/* Sanity check */
|
||||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||||
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
||||||
|
|
||||||
/* Decompress : frame header; part 1 */
|
|
||||||
switch (dctx->stage)
|
switch (dctx->stage)
|
||||||
{
|
{
|
||||||
case ZSTDds_getFrameHeaderSize :
|
case ZSTDds_getFrameHeaderSize :
|
||||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||||
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
|
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||||
|
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
|
||||||
|
dctx->stage = ZSTDds_decodeSkippableHeader;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
||||||
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
|
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
|
||||||
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||||
@ -1030,6 +1109,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||||
if (bp.blockType == bt_end) {
|
if (bp.blockType == bt_end) {
|
||||||
|
if (dctx->fParams.checksumFlag) {
|
||||||
|
U64 const h64 = XXH64_digest(&dctx->xxhState);
|
||||||
|
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
|
||||||
|
const BYTE* const ip = (const BYTE*)src;
|
||||||
|
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
|
||||||
|
if (check32 != h32) return ERROR(checksum_wrong);
|
||||||
|
}
|
||||||
dctx->expected = 0;
|
dctx->expected = 0;
|
||||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||||
} else {
|
} else {
|
||||||
@ -1061,8 +1147,21 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||||
dctx->expected = ZSTD_blockHeaderSize;
|
dctx->expected = ZSTD_blockHeaderSize;
|
||||||
dctx->previousDstEnd = (char*)dst + rSize;
|
dctx->previousDstEnd = (char*)dst + rSize;
|
||||||
|
if (ZSTD_isError(rSize)) return rSize;
|
||||||
|
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
||||||
return rSize;
|
return rSize;
|
||||||
}
|
}
|
||||||
|
case ZSTDds_decodeSkippableHeader:
|
||||||
|
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
|
||||||
|
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
|
||||||
|
dctx->stage = ZSTDds_skipFrame;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
case ZSTDds_skipFrame:
|
||||||
|
{ dctx->expected = 0;
|
||||||
|
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
return ERROR(GENERIC); /* impossible */
|
return ERROR(GENERIC); /* impossible */
|
||||||
}
|
}
|
||||||
@ -1077,18 +1176,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
|||||||
dctx->previousDstEnd = (const char*)dict + dictSize;
|
dctx->previousDstEnd = (const char*)dict + dictSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart)
|
||||||
{
|
{
|
||||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
|
size_t dictSize = dictSizeStart;
|
||||||
|
|
||||||
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
{ size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||||
dict = (const char*)dict + hSize;
|
dict = (const char*)dict + hSize;
|
||||||
dictSize -= hSize;
|
dictSize -= hSize;
|
||||||
|
}
|
||||||
|
|
||||||
{ short offcodeNCount[MaxOff+1];
|
{ short offcodeNCount[MaxOff+1];
|
||||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
@ -1098,7 +1198,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
|
|||||||
|
|
||||||
{ short matchlengthNCount[MaxML+1];
|
{ short matchlengthNCount[MaxML+1];
|
||||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
@ -1108,37 +1208,42 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
|
|||||||
|
|
||||||
{ short litlengthNCount[MaxLL+1];
|
{ short litlengthNCount[MaxLL+1];
|
||||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||||
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||||
|
dictSize -= litlengthHeaderSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
dctx->flagRepeatTable = 1;
|
dctx->flagRepeatTable = 1;
|
||||||
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
return dictSizeStart - dictSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
size_t eSize;
|
if (dictSize < 8) return ERROR(dictionary_corrupted);
|
||||||
U32 const magic = MEM_readLE32(dict);
|
{ U32 const magic = MEM_readLE32(dict);
|
||||||
if (magic != ZSTD_DICT_MAGIC) {
|
if (magic != ZSTD_DICT_MAGIC) {
|
||||||
/* pure content mode */
|
/* pure content mode */
|
||||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
/* load entropy tables */
|
dctx->dictID = MEM_readLE32((const char*)dict + 4);
|
||||||
dict = (const char*)dict + 4;
|
|
||||||
dictSize -= 4;
|
|
||||||
eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
|
|
||||||
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
|
|
||||||
|
|
||||||
/* reference dictionary content */
|
/* load entropy tables */
|
||||||
|
dict = (const char*)dict + 8;
|
||||||
|
dictSize -= 8;
|
||||||
|
{ size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
|
||||||
|
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||||
dict = (const char*)dict + eSize;
|
dict = (const char*)dict + eSize;
|
||||||
dictSize -= eSize;
|
dictSize -= eSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* reference dictionary content */
|
||||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1154,4 +1259,3 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
|
|||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,11 +59,13 @@
|
|||||||
#include "mem.h" /* read */
|
#include "mem.h" /* read */
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
#include "fse.h"
|
#include "fse.h"
|
||||||
#include "huf_static.h"
|
#define HUF_STATIC_LINKING_ONLY
|
||||||
#include "zstd_internal.h"
|
#include "huf.h"
|
||||||
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
|
#include "xxhash.h"
|
||||||
#include "divsufsort.h"
|
#include "divsufsort.h"
|
||||||
#include "zdict_static.h"
|
#define ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#include "zdict.h"
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
@ -104,8 +106,7 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
|
|||||||
{
|
{
|
||||||
const BYTE* const b = (const BYTE*)ptr;
|
const BYTE* const b = (const BYTE*)ptr;
|
||||||
size_t u;
|
size_t u;
|
||||||
for (u=0; u<length; u++)
|
for (u=0; u<length; u++) {
|
||||||
{
|
|
||||||
BYTE c = b[u];
|
BYTE c = b[u];
|
||||||
if (c<32 || c>126) c = '.'; /* non-printable char */
|
if (c<32 || c>126) c = '.'; /* non-printable char */
|
||||||
DISPLAYLEVEL(dlevel, "%c", c);
|
DISPLAYLEVEL(dlevel, "%c", c);
|
||||||
@ -198,8 +199,12 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch)
|
|||||||
{
|
{
|
||||||
const char* const pStart = (const char*)pIn;
|
const char* const pStart = (const char*)pIn;
|
||||||
for (;;) {
|
for (;;) {
|
||||||
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||||
if (!diff) { pIn = (const char*)pIn+sizeof(size_t); pMatch = (const char*)pMatch+sizeof(size_t); continue; }
|
if (!diff) {
|
||||||
|
pIn = (const char*)pIn+sizeof(size_t);
|
||||||
|
pMatch = (const char*)pMatch+sizeof(size_t);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
|
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
|
||||||
return (size_t)((const char*)pIn - pStart);
|
return (size_t)((const char*)pIn - pStart);
|
||||||
}
|
}
|
||||||
@ -346,9 +351,8 @@ static dictItem ZDICT_analyzePos(
|
|||||||
maxLength = i;
|
maxLength = i;
|
||||||
|
|
||||||
/* reduce maxLength in case of final into repetitive data */
|
/* reduce maxLength in case of final into repetitive data */
|
||||||
{
|
{ U32 l = (U32)maxLength;
|
||||||
U32 l = (U32)maxLength;
|
BYTE const c = b[pos + maxLength-1];
|
||||||
BYTE c = b[pos + maxLength-1];
|
|
||||||
while (b[pos+l-2]==c) l--;
|
while (b[pos+l-2]==c) l--;
|
||||||
maxLength = l;
|
maxLength = l;
|
||||||
}
|
}
|
||||||
@ -367,12 +371,10 @@ static dictItem ZDICT_analyzePos(
|
|||||||
solution.savings = savings[maxLength];
|
solution.savings = savings[maxLength];
|
||||||
|
|
||||||
/* mark positions done */
|
/* mark positions done */
|
||||||
{
|
{ U32 id;
|
||||||
U32 id;
|
|
||||||
U32 testedPos;
|
|
||||||
for (id=start; id<end; id++) {
|
for (id=start; id<end; id++) {
|
||||||
U32 p, pEnd;
|
U32 p, pEnd;
|
||||||
testedPos = suffix[id];
|
U32 const testedPos = suffix[id];
|
||||||
if (testedPos == pos)
|
if (testedPos == pos)
|
||||||
length = solution.length;
|
length = solution.length;
|
||||||
else {
|
else {
|
||||||
@ -439,7 +441,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
|
|||||||
static void ZDICT_removeDictItem(dictItem* table, U32 id)
|
static void ZDICT_removeDictItem(dictItem* table, U32 id)
|
||||||
{
|
{
|
||||||
/* convention : first element is nb of elts */
|
/* convention : first element is nb of elts */
|
||||||
U32 max = table->pos;
|
U32 const max = table->pos;
|
||||||
U32 u;
|
U32 u;
|
||||||
if (!id) return; /* protection, should never happen */
|
if (!id) return; /* protection, should never happen */
|
||||||
for (u=id; u<max-1; u++)
|
for (u=id; u<max-1; u++)
|
||||||
@ -463,8 +465,7 @@ static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* insert */
|
/* insert */
|
||||||
{
|
{ U32 current;
|
||||||
U32 current;
|
|
||||||
U32 nextElt = table->pos;
|
U32 nextElt = table->pos;
|
||||||
if (nextElt >= maxSize) nextElt = maxSize-1;
|
if (nextElt >= maxSize) nextElt = maxSize-1;
|
||||||
current = nextElt-1;
|
current = nextElt-1;
|
||||||
@ -530,8 +531,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
|||||||
DISPLAYLEVEL(2, "finding patterns ... \n");
|
DISPLAYLEVEL(2, "finding patterns ... \n");
|
||||||
DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio);
|
DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio);
|
||||||
|
|
||||||
{
|
{ U32 cursor; for (cursor=0; cursor < bufferSize; ) {
|
||||||
U32 cursor; for (cursor=0; cursor < bufferSize; ) {
|
|
||||||
dictItem solution;
|
dictItem solution;
|
||||||
if (doneMarks[cursor]) { cursor++; continue; }
|
if (doneMarks[cursor]) { cursor++; continue; }
|
||||||
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
|
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
|
||||||
@ -542,8 +542,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
|||||||
} }
|
} }
|
||||||
|
|
||||||
/* limit dictionary size */
|
/* limit dictionary size */
|
||||||
{
|
{ U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */
|
||||||
U32 max = dictList->pos; /* convention : nb of useful elts within dictList */
|
|
||||||
U32 currentSize = 0;
|
U32 currentSize = 0;
|
||||||
U32 n; for (n=1; n<max; n++) {
|
U32 n; for (n=1; n<max; n++) {
|
||||||
currentSize += dictList[n].length;
|
currentSize += dictList[n].length;
|
||||||
@ -785,7 +784,7 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
|||||||
{
|
{
|
||||||
char* dstPtr = (char*)dictBuffer + dictSize;
|
char* dstPtr = (char*)dictBuffer + dictSize;
|
||||||
const char* srcPtr = (const char*)samplesBuffer;
|
const char* srcPtr = (const char*)samplesBuffer;
|
||||||
size_t nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
|
size_t const nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
|
||||||
size_t segNb, interSize;
|
size_t segNb, interSize;
|
||||||
|
|
||||||
if (nbSegments <= 2) return ERROR(srcSize_wrong);
|
if (nbSegments <= 2) return ERROR(srcSize_wrong);
|
||||||
@ -810,6 +809,33 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
|||||||
return nbSegments * DIB_FASTSEGMENTSIZE;
|
return nbSegments * DIB_FASTSEGMENTSIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||||
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||||
|
ZDICT_params_t params)
|
||||||
|
{
|
||||||
|
size_t hSize;
|
||||||
|
unsigned const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
|
||||||
|
|
||||||
|
/* dictionary header */
|
||||||
|
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
|
||||||
|
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
|
||||||
|
U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11);
|
||||||
|
MEM_writeLE32((char*)dictBuffer+4, dictID);
|
||||||
|
}
|
||||||
|
hSize = 8;
|
||||||
|
|
||||||
|
/* entropy tables */
|
||||||
|
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
||||||
|
DISPLAYLEVEL(2, "statistics ... \n");
|
||||||
|
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
|
||||||
|
compressionLevel,
|
||||||
|
samplesBuffer, samplesSizes, nbSamples,
|
||||||
|
(char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
|
||||||
|
|
||||||
|
if (hSize + dictContentSize < dictBufferCapacity)
|
||||||
|
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
|
||||||
|
return MIN(dictBufferCapacity, hSize+dictContentSize);
|
||||||
|
}
|
||||||
|
|
||||||
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
|
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
|
||||||
/*! ZDICT_trainFromBuffer_unsafe() :
|
/*! ZDICT_trainFromBuffer_unsafe() :
|
||||||
@ -818,14 +844,13 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
|||||||
*/
|
*/
|
||||||
size_t ZDICT_trainFromBuffer_unsafe(
|
size_t ZDICT_trainFromBuffer_unsafe(
|
||||||
void* dictBuffer, size_t maxDictSize,
|
void* dictBuffer, size_t maxDictSize,
|
||||||
const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples,
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||||
ZDICT_params_t params)
|
ZDICT_params_t params)
|
||||||
{
|
{
|
||||||
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
|
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
|
||||||
dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
|
dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
|
||||||
unsigned selectivity = params.selectivityLevel;
|
unsigned selectivity = params.selectivityLevel;
|
||||||
unsigned compressionLevel = params.compressionLevel;
|
size_t const targetDictSize = maxDictSize;
|
||||||
size_t targetDictSize = maxDictSize;
|
|
||||||
size_t sBuffSize;
|
size_t sBuffSize;
|
||||||
size_t dictSize = 0;
|
size_t dictSize = 0;
|
||||||
|
|
||||||
@ -834,18 +859,17 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
|||||||
if (!dictList) return ERROR(memory_allocation);
|
if (!dictList) return ERROR(memory_allocation);
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
|
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
|
||||||
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
|
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
|
||||||
ZDICT_initDictItem(dictList);
|
ZDICT_initDictItem(dictList);
|
||||||
g_displayLevel = params.notificationLevel;
|
g_displayLevel = params.notificationLevel;
|
||||||
if (selectivity==0) selectivity = g_selectivity_default;
|
if (selectivity==0) selectivity = g_selectivity_default;
|
||||||
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
|
|
||||||
|
|
||||||
/* build dictionary */
|
/* build dictionary */
|
||||||
if (selectivity>1) { /* selectivity == 1 => fast mode */
|
if (selectivity>1) { /* selectivity == 1 => fast mode */
|
||||||
ZDICT_trainBuffer(dictList, dictListSize,
|
ZDICT_trainBuffer(dictList, dictListSize,
|
||||||
samplesBuffer, sBuffSize,
|
samplesBuffer, sBuffSize,
|
||||||
sampleSizes, nbSamples,
|
samplesSizes, nbSamples,
|
||||||
selectivity, (U32)targetDictSize);
|
selectivity, (U32)targetDictSize);
|
||||||
|
|
||||||
/* display best matches */
|
/* display best matches */
|
||||||
@ -867,18 +891,16 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
|||||||
|
|
||||||
/* create dictionary */
|
/* create dictionary */
|
||||||
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||||
size_t hSize;
|
|
||||||
BYTE* ptr;
|
|
||||||
U32 u;
|
|
||||||
|
|
||||||
/* build dict content */
|
/* build dict content */
|
||||||
ptr = (BYTE*)dictBuffer + maxDictSize;
|
{ U32 u;
|
||||||
|
BYTE* ptr = (BYTE*)dictBuffer + maxDictSize;
|
||||||
for (u=1; u<dictList->pos; u++) {
|
for (u=1; u<dictList->pos; u++) {
|
||||||
U32 l = dictList[u].length;
|
U32 l = dictList[u].length;
|
||||||
ptr -= l;
|
ptr -= l;
|
||||||
if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
|
if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
|
||||||
memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
|
memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
|
||||||
}
|
} }
|
||||||
|
|
||||||
/* fast mode dict content */
|
/* fast mode dict content */
|
||||||
if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */
|
if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */
|
||||||
@ -888,21 +910,9 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
|||||||
samplesBuffer, sBuffSize);
|
samplesBuffer, sBuffSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* dictionary header */
|
dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize,
|
||||||
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
|
samplesBuffer, samplesSizes, nbSamples,
|
||||||
hSize = 4;
|
params);
|
||||||
|
|
||||||
/* entropic tables */
|
|
||||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
|
||||||
DISPLAYLEVEL(2, "statistics ... \n");
|
|
||||||
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+4, maxDictSize-4,
|
|
||||||
compressionLevel,
|
|
||||||
samplesBuffer, sampleSizes, nbSamples,
|
|
||||||
(char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
|
|
||||||
|
|
||||||
if (hSize + dictContentSize < maxDictSize)
|
|
||||||
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
|
|
||||||
dictSize = MIN(maxDictSize, hSize+dictContentSize);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* clean up */
|
/* clean up */
|
||||||
@ -947,3 +957,12 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
|||||||
params);
|
params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||||
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
||||||
|
{
|
||||||
|
ZDICT_params_t params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
|
return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity,
|
||||||
|
samplesBuffer, samplesSizes, nbSamples,
|
||||||
|
params);
|
||||||
|
}
|
||||||
|
@ -52,6 +52,21 @@ extern "C" {
|
|||||||
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||||
|
|
||||||
|
/*! ZDICT_addEntropyTablesFromBuffer() :
|
||||||
|
|
||||||
|
Given a content-only dictionary (built for example from common strings in
|
||||||
|
the input), add entropy tables computed from the memory buffer
|
||||||
|
`samplesBuffer`, where `nbSamples` samples have been stored concatenated.
|
||||||
|
Each sample size is provided into an orderly table `samplesSizes`.
|
||||||
|
|
||||||
|
The input dictionary is the last `dictContentSize` bytes of `dictBuffer`. The
|
||||||
|
resulting dictionary with added entropy tables will written back to
|
||||||
|
`dictBuffer`.
|
||||||
|
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
|
||||||
|
*/
|
||||||
|
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||||
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
* Helper functions
|
* Helper functions
|
||||||
@ -60,6 +75,44 @@ unsigned ZDICT_isError(size_t errorCode);
|
|||||||
const char* ZDICT_getErrorName(size_t errorCode);
|
const char* ZDICT_getErrorName(size_t errorCode);
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef ZDICT_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
/* ====================================================================================
|
||||||
|
* The definitions in this section are considered experimental.
|
||||||
|
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||||
|
* They are provided for advanced usages.
|
||||||
|
* Use them only in association with static linking.
|
||||||
|
* ==================================================================================== */
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Public type
|
||||||
|
***************************************/
|
||||||
|
typedef struct {
|
||||||
|
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
|
||||||
|
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
|
||||||
|
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||||
|
unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
|
||||||
|
unsigned reserved[2]; /* space for future parameters */
|
||||||
|
} ZDICT_params_t;
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Public functions
|
||||||
|
***************************************/
|
||||||
|
/*! ZDICT_trainFromBuffer_advanced() :
|
||||||
|
Same as ZDICT_trainFromBuffer() with control over more parameters.
|
||||||
|
`parameters` is optional and can be provided with values set to 0 to mean "default".
|
||||||
|
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
|
||||||
|
or an error code, which can be tested by ZDICT_isError().
|
||||||
|
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
|
||||||
|
*/
|
||||||
|
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||||
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||||
|
ZDICT_params_t parameters);
|
||||||
|
|
||||||
|
#endif /* ZDICT_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -1,80 +0,0 @@
|
|||||||
/*
|
|
||||||
dictBuilder header file
|
|
||||||
for static linking only
|
|
||||||
Copyright (C) Yann Collet 2016
|
|
||||||
|
|
||||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
|
||||||
modification, are permitted provided that the following conditions are
|
|
||||||
met:
|
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
|
||||||
notice, this list of conditions and the following disclaimer.
|
|
||||||
* Redistributions in binary form must reproduce the above
|
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
|
||||||
in the documentation and/or other materials provided with the
|
|
||||||
distribution.
|
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
|
|
||||||
You can contact the author at :
|
|
||||||
- Zstd source repository : https://www.zstd.net
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* This library is EXPERIMENTAL, below API is not yet stable */
|
|
||||||
|
|
||||||
#ifndef DICTBUILDER_STATIC_H_002
|
|
||||||
#define DICTBUILDER_STATIC_H_002
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Dependencies
|
|
||||||
***************************************/
|
|
||||||
#include "zdict.h"
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Public type
|
|
||||||
***************************************/
|
|
||||||
typedef struct {
|
|
||||||
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
|
|
||||||
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
|
|
||||||
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
|
||||||
unsigned reserved[3]; /* space for future parameters */
|
|
||||||
} ZDICT_params_t;
|
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
|
||||||
* Public functions
|
|
||||||
***************************************/
|
|
||||||
/*! ZDICT_trainFromBuffer_advanced() :
|
|
||||||
Same as ZDICT_trainFromBuffer() with control over more parameters.
|
|
||||||
`parameters` is optional and can be provided with values set to 0 to mean "default".
|
|
||||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
|
|
||||||
or an error code, which can be tested by DiB_isError().
|
|
||||||
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
|
|
||||||
*/
|
|
||||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
|
||||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
|
||||||
ZDICT_params_t parameters);
|
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif /* DICTBUILDER_STATIC_H_002 */
|
|
@ -1,3 +0,0 @@
|
|||||||
-I../lib/common
|
|
||||||
-I../lib/legacy
|
|
||||||
-I./legacy
|
|
5
programs/.gitignore
vendored
5
programs/.gitignore
vendored
@ -1,6 +1,8 @@
|
|||||||
# local binary (Makefile)
|
# local binary (Makefile)
|
||||||
zstd
|
zstd
|
||||||
zstd32
|
zstd32
|
||||||
|
zstd-compress
|
||||||
|
zstd-decompress
|
||||||
fullbench
|
fullbench
|
||||||
fullbench32
|
fullbench32
|
||||||
fuzzer
|
fuzzer
|
||||||
@ -38,6 +40,9 @@ roundTripCrash
|
|||||||
dictionary
|
dictionary
|
||||||
grillResults.txt
|
grillResults.txt
|
||||||
_*
|
_*
|
||||||
|
tmp*
|
||||||
|
*.zst
|
||||||
|
result
|
||||||
|
|
||||||
# fuzzer
|
# fuzzer
|
||||||
afl
|
afl
|
||||||
|
@ -33,29 +33,31 @@
|
|||||||
|
|
||||||
DESTDIR?=
|
DESTDIR?=
|
||||||
PREFIX ?= /usr/local
|
PREFIX ?= /usr/local
|
||||||
CPPFLAGS= -I../lib/common
|
|
||||||
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
|
|
||||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
|
||||||
|
|
||||||
BINDIR = $(PREFIX)/bin
|
BINDIR = $(PREFIX)/bin
|
||||||
MANDIR = $(PREFIX)/share/man/man1
|
MANDIR = $(PREFIX)/share/man/man1
|
||||||
|
|
||||||
ZSTDDIR = ../lib
|
ZSTDDIR = ../lib
|
||||||
|
|
||||||
ZSTDCOMP_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
|
CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
|
||||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
|
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
||||||
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||||
|
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||||
|
|
||||||
|
|
||||||
|
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||||
|
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
|
||||||
|
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/decompress/zstd_decompress.c
|
||||||
|
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||||
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
|
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
|
||||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES)
|
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
|
||||||
|
|
||||||
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||||
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
||||||
ZSTD_FILES_LEGACY:=
|
ZSTDLEGACY_FILES:=
|
||||||
else
|
else
|
||||||
ZSTD_LEGACY_SUPPORT:=1
|
ZSTD_LEGACY_SUPPORT:=1
|
||||||
CPPFLAGS += -I../lib/legacy -I./legacy
|
CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy
|
||||||
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
|
ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c
|
||||||
$(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
|
||||||
@ -78,12 +80,12 @@ default: zstd
|
|||||||
|
|
||||||
all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32
|
all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32
|
||||||
|
|
||||||
zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||||
zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c
|
zstdcli.c fileio.c bench.c datagen.c dibio.c
|
||||||
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
||||||
|
|
||||||
zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||||
zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c
|
zstdcli.c fileio.c bench.c datagen.c dibio.c
|
||||||
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
||||||
|
|
||||||
zstd_nolegacy :
|
zstd_nolegacy :
|
||||||
@ -103,10 +105,12 @@ zstd-pgo : clean zstd
|
|||||||
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
|
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
|
||||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
|
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
|
||||||
|
|
||||||
zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
|
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
|
||||||
|
$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
|
||||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||||
|
|
||||||
zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
|
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
|
||||||
|
$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
|
||||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||||
|
|
||||||
zstd-small: clean
|
zstd-small: clean
|
||||||
@ -118,23 +122,21 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
|||||||
fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
fuzzer : $(ZSTD_FILES) \
|
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||||
datagen.c xxhash.c fuzzer.c
|
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
fuzzer32: $(ZSTD_FILES) \
|
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||||
datagen.c xxhash.c fuzzer.c
|
fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \
|
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||||
datagen.c xxhash.c zbufftest.c
|
|
||||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \
|
zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||||
datagen.c xxhash.c zbufftest.c
|
|
||||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||||
|
|
||||||
paramgrill : $(ZSTD_FILES) datagen.c xxhash.c paramgrill.c
|
paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
|
||||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||||
|
|
||||||
datagen : datagen.c datagencli.c
|
datagen : datagen.c datagencli.c
|
||||||
@ -206,7 +208,7 @@ endif
|
|||||||
#------------------------------------------------------------------------
|
#------------------------------------------------------------------------
|
||||||
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||||
zstd-playTests: datagen
|
zstd-playTests: datagen
|
||||||
ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST)
|
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
|
||||||
|
|
||||||
test: test-zstd test-fullbench test-fuzzer test-zbuff
|
test: test-zstd test-fullbench test-fuzzer test-zbuff
|
||||||
|
|
||||||
|
@ -32,7 +32,8 @@
|
|||||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||||
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "zstd_static.h"
|
#define ZSTD_STATIC_LINKING_ONLY
|
||||||
|
#include "zstd.h"
|
||||||
#include "datagen.h" /* RDG_genBuffer */
|
#include "datagen.h" /* RDG_genBuffer */
|
||||||
#include "xxhash.h"
|
#include "xxhash.h"
|
||||||
|
|
||||||
@ -220,7 +221,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
{ ZSTD_parameters params;
|
{ ZSTD_parameters params;
|
||||||
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
|
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize);
|
|
||||||
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
|
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
|
||||||
if (ZSTD_isError(initResult)) break;
|
if (ZSTD_isError(initResult)) break;
|
||||||
} }
|
} }
|
||||||
@ -505,4 +505,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -21,10 +21,11 @@
|
|||||||
You can contact the author at :
|
You can contact the author at :
|
||||||
- ZSTD homepage : http://www.zstd.net/
|
- ZSTD homepage : http://www.zstd.net/
|
||||||
*/
|
*/
|
||||||
#pragma once
|
#ifndef BENCH_H_121279284357
|
||||||
|
#define BENCH_H_121279284357
|
||||||
|
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
/* Main function */
|
|
||||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||||
const char* dictFileName, int cLevel, int cLevelLast);
|
const char* dictFileName, int cLevel, int cLevelLast);
|
||||||
|
|
||||||
@ -34,3 +35,4 @@ void BMK_SetBlockSize(size_t blockSize);
|
|||||||
void BMK_setAdditionalParam(int additionalParam);
|
void BMK_setAdditionalParam(int additionalParam);
|
||||||
void BMK_setNotificationLevel(unsigned level);
|
void BMK_setNotificationLevel(unsigned level);
|
||||||
|
|
||||||
|
#endif /* BENCH_H_121279284357 */
|
||||||
|
@ -101,27 +101,30 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo
|
|||||||
/* ********************************************************
|
/* ********************************************************
|
||||||
* File related operations
|
* File related operations
|
||||||
**********************************************************/
|
**********************************************************/
|
||||||
static void DiB_loadFiles(void* buffer, size_t bufferSize,
|
/** DiB_loadFiles() :
|
||||||
|
* @return : nb of files effectively loaded into `buffer` */
|
||||||
|
static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
|
||||||
size_t* fileSizes,
|
size_t* fileSizes,
|
||||||
const char** fileNamesTable, unsigned nbFiles)
|
const char** fileNamesTable, unsigned nbFiles)
|
||||||
{
|
{
|
||||||
char* buff = (char*)buffer;
|
char* const buff = (char*)buffer;
|
||||||
size_t pos = 0;
|
size_t pos = 0;
|
||||||
unsigned n;
|
unsigned n;
|
||||||
|
|
||||||
for (n=0; n<nbFiles; n++) {
|
for (n=0; n<nbFiles; n++) {
|
||||||
size_t readSize;
|
unsigned long long const fs64 = UTIL_getFileSize(fileNamesTable[n]);
|
||||||
unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
size_t const fileSize = (size_t)(fs64 > bufferSize-pos ? 0 : fs64);
|
||||||
FILE* f = fopen(fileNamesTable[n], "rb");
|
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||||
if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */
|
{ size_t const readSize = fread(buff+pos, 1, fileSize, f);
|
||||||
readSize = fread(buff+pos, 1, (size_t)fileSize, f);
|
if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
pos += readSize; }
|
||||||
pos += readSize;
|
fileSizes[n] = fileSize;
|
||||||
fileSizes[n] = (size_t)fileSize;
|
|
||||||
fclose(f);
|
fclose(f);
|
||||||
|
if (fileSize == 0) break; /* stop there, not enough memory to load all files */
|
||||||
}
|
}
|
||||||
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -130,7 +133,7 @@ static void DiB_loadFiles(void* buffer, size_t bufferSize,
|
|||||||
**********************************************************/
|
**********************************************************/
|
||||||
static size_t DiB_findMaxMem(unsigned long long requiredMem)
|
static size_t DiB_findMaxMem(unsigned long long requiredMem)
|
||||||
{
|
{
|
||||||
size_t step = 8 MB;
|
size_t const step = 8 MB;
|
||||||
void* testmem = NULL;
|
void* testmem = NULL;
|
||||||
|
|
||||||
requiredMem = (((requiredMem >> 23) + 1) << 23);
|
requiredMem = (((requiredMem >> 23) + 1) << 23);
|
||||||
@ -162,7 +165,7 @@ static void DiB_fillNoise(void* buffer, size_t length)
|
|||||||
static void DiB_saveDict(const char* dictFileName,
|
static void DiB_saveDict(const char* dictFileName,
|
||||||
const void* buff, size_t buffSize)
|
const void* buff, size_t buffSize)
|
||||||
{
|
{
|
||||||
FILE* f = fopen(dictFileName, "wb");
|
FILE* const f = fopen(dictFileName, "wb");
|
||||||
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
|
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
|
||||||
|
|
||||||
{ size_t const n = fwrite(buff, 1, buffSize, f);
|
{ size_t const n = fwrite(buff, 1, buffSize, f);
|
||||||
@ -185,35 +188,32 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
|
|||||||
ZDICT_params_t parameters);
|
ZDICT_params_t parameters);
|
||||||
|
|
||||||
|
|
||||||
|
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||||
const char** fileNamesTable, unsigned nbFiles,
|
const char** fileNamesTable, unsigned nbFiles,
|
||||||
ZDICT_params_t params)
|
ZDICT_params_t params)
|
||||||
{
|
{
|
||||||
void* srcBuffer;
|
void* const dictBuffer = malloc(maxDictSize);
|
||||||
size_t benchedSize;
|
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||||
unsigned long long totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
||||||
void* dictBuffer = malloc(maxDictSize);
|
size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
|
||||||
size_t dictSize;
|
void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
|
||||||
int result = 0;
|
int result = 0;
|
||||||
|
|
||||||
|
/* Checks */
|
||||||
|
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
g_displayLevel = params.notificationLevel;
|
g_displayLevel = params.notificationLevel;
|
||||||
benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
|
||||||
if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
|
||||||
if (benchedSize < totalSizeToLoad)
|
if (benchedSize < totalSizeToLoad)
|
||||||
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
|
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
|
||||||
|
|
||||||
/* Memory allocation & restrictions */
|
|
||||||
srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */
|
|
||||||
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
|
||||||
|
|
||||||
/* Load input buffer */
|
/* Load input buffer */
|
||||||
DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||||
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||||
|
|
||||||
/* call buffer version */
|
{ size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
||||||
dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
|
||||||
srcBuffer, fileSizes, nbFiles,
|
srcBuffer, fileSizes, nbFiles,
|
||||||
params);
|
params);
|
||||||
if (ZDICT_isError(dictSize)) {
|
if (ZDICT_isError(dictSize)) {
|
||||||
@ -221,10 +221,10 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
result = 1;
|
result = 1;
|
||||||
goto _cleanup;
|
goto _cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* save dict */
|
/* save dict */
|
||||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||||
DiB_saveDict(dictFileName, dictBuffer, dictSize);
|
DiB_saveDict(dictFileName, dictBuffer, dictSize);
|
||||||
|
}
|
||||||
|
|
||||||
/* clean up */
|
/* clean up */
|
||||||
_cleanup:
|
_cleanup:
|
||||||
|
@ -32,7 +32,8 @@
|
|||||||
/*-*************************************
|
/*-*************************************
|
||||||
* Dependencies
|
* Dependencies
|
||||||
***************************************/
|
***************************************/
|
||||||
#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */
|
#define ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#include "zdict.h" /* ZDICT_params_t */
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
|
@ -56,9 +56,11 @@
|
|||||||
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "fileio.h"
|
#include "fileio.h"
|
||||||
#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||||
|
#include "zstd.h"
|
||||||
#include "zstd_internal.h" /* MIN, KB, MB */
|
#include "zstd_internal.h" /* MIN, KB, MB */
|
||||||
#include "zbuff_static.h"
|
#define ZBUFF_STATIC_LINKING_ONLY
|
||||||
|
#include "zbuff.h"
|
||||||
|
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||||
# include "zstd_legacy.h" /* ZSTD_isLegacy */
|
# include "zstd_legacy.h" /* ZSTD_isLegacy */
|
||||||
@ -97,7 +99,7 @@
|
|||||||
|
|
||||||
#define CACHELINE 64
|
#define CACHELINE 64
|
||||||
|
|
||||||
#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */
|
#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
|
||||||
|
|
||||||
#define FNSPACE 30
|
#define FNSPACE 30
|
||||||
|
|
||||||
@ -133,6 +135,10 @@ static U32 g_maxWLog = 23;
|
|||||||
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
|
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
|
||||||
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
||||||
|
static U32 g_dictIDFlag = 1;
|
||||||
|
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
||||||
|
static U32 g_checksumFlag = 0;
|
||||||
|
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
@ -186,7 +192,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
|||||||
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (!g_overwrite) { /* Check if destination file already exists */
|
if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
|
||||||
f = fopen( dstFileName, "rb" );
|
f = fopen( dstFileName, "rb" );
|
||||||
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
|
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
|
||||||
fclose(f);
|
fclose(f);
|
||||||
@ -212,12 +218,12 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
|||||||
/*! FIO_loadFile() :
|
/*! FIO_loadFile() :
|
||||||
* creates a buffer, pointed by `*bufferPtr`,
|
* creates a buffer, pointed by `*bufferPtr`,
|
||||||
* loads `filename` content into it,
|
* loads `filename` content into it,
|
||||||
* up to MAX_DICT_SIZE bytes
|
* up to MAX_DICT_SIZE bytes.
|
||||||
|
* @return : loaded size
|
||||||
*/
|
*/
|
||||||
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
|
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
|
||||||
{
|
{
|
||||||
FILE* fileHandle;
|
FILE* fileHandle;
|
||||||
size_t readSize;
|
|
||||||
U64 fileSize;
|
U64 fileSize;
|
||||||
|
|
||||||
*bufferPtr = NULL;
|
*bufferPtr = NULL;
|
||||||
@ -237,8 +243,8 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
|
|||||||
}
|
}
|
||||||
*bufferPtr = (BYTE*)malloc((size_t)fileSize);
|
*bufferPtr = (BYTE*)malloc((size_t)fileSize);
|
||||||
if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
|
if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
|
||||||
readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||||
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName);
|
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
|
||||||
fclose(fileHandle);
|
fclose(fileHandle);
|
||||||
return (size_t)fileSize;
|
return (size_t)fileSize;
|
||||||
}
|
}
|
||||||
@ -308,12 +314,18 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
|||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
{ ZSTD_parameters params;
|
{ ZSTD_parameters params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
|
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
|
params.fParams.checksumFlag = g_checksumFlag;
|
||||||
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||||
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
|
if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) {
|
||||||
|
params.cParams.windowLog = g_maxWLog;
|
||||||
|
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
|
||||||
}
|
}
|
||||||
|
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
||||||
|
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
|
||||||
|
} }
|
||||||
|
|
||||||
/* Main compression loop */
|
/* Main compression loop */
|
||||||
readsize = 0;
|
readsize = 0;
|
||||||
@ -409,14 +421,10 @@ static int FIO_compressFilename_extRess(cRess_t ress,
|
|||||||
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||||
const char* dictFileName, int compressionLevel)
|
const char* dictFileName, int compressionLevel)
|
||||||
{
|
{
|
||||||
clock_t start;
|
clock_t const start = clock();
|
||||||
cRess_t ress;
|
cRess_t const ress = FIO_createCResources(dictFileName);
|
||||||
int issueWithSrcFile = 0;
|
int issueWithSrcFile = 0;
|
||||||
|
|
||||||
/* Init */
|
|
||||||
start = clock();
|
|
||||||
ress = FIO_createCResources(dictFileName);
|
|
||||||
|
|
||||||
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
|
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
|
||||||
|
|
||||||
FIO_freeCResources(ress);
|
FIO_freeCResources(ress);
|
||||||
@ -607,7 +615,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
|||||||
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||||
|
|
||||||
/* Header loading (optional, saves one loop) */
|
/* Header loading (optional, saves one loop) */
|
||||||
{ size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
|
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
|
||||||
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
|
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
|
||||||
readSize = alreadyLoaded + loadedSize;
|
readSize = alreadyLoaded + loadedSize;
|
||||||
}
|
}
|
||||||
@ -689,7 +697,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
if (magic != ZSTD_MAGICNUMBER) {
|
if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
|
||||||
if (g_overwrite) /* -df : pass-through mode */
|
if (g_overwrite) /* -df : pass-through mode */
|
||||||
return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
||||||
else {
|
else {
|
||||||
@ -701,7 +709,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
|||||||
|
|
||||||
/* Final Status */
|
/* Final Status */
|
||||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||||
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
|
DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
|
||||||
|
|
||||||
/* Close */
|
/* Close */
|
||||||
fclose(srcFile);
|
fclose(srcFile);
|
||||||
|
@ -48,6 +48,8 @@ void FIO_overwriteMode(void);
|
|||||||
void FIO_setNotificationLevel(unsigned level);
|
void FIO_setNotificationLevel(unsigned level);
|
||||||
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
||||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||||
|
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||||
|
void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
|
@ -28,12 +28,13 @@
|
|||||||
#include "util.h" /* Compiler options, UTIL_GetFileSize */
|
#include "util.h" /* Compiler options, UTIL_GetFileSize */
|
||||||
#include <stdlib.h> /* malloc */
|
#include <stdlib.h> /* malloc */
|
||||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||||
#include <string.h> /* strcmp */
|
|
||||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||||
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "zstd_static.h" /* ZSTD_VERSION_STRING */
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */
|
||||||
#include "fse_static.h"
|
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||||
|
#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */
|
||||||
|
#include "fse.h"
|
||||||
#include "zbuff.h"
|
#include "zbuff.h"
|
||||||
#include "datagen.h"
|
#include "datagen.h"
|
||||||
|
|
||||||
@ -544,4 +545,3 @@ int main(int argc, const char** argv)
|
|||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -28,7 +28,7 @@
|
|||||||
#ifdef _MSC_VER /* Visual Studio */
|
#ifdef _MSC_VER /* Visual Studio */
|
||||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
@ -40,7 +40,9 @@
|
|||||||
#include <sys/timeb.h> /* timeb */
|
#include <sys/timeb.h> /* timeb */
|
||||||
#include <string.h> /* strcmp */
|
#include <string.h> /* strcmp */
|
||||||
#include <time.h> /* clock_t */
|
#include <time.h> /* clock_t */
|
||||||
#include "zstd_static.h" /* ZSTD_VERSION_STRING */
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */
|
||||||
|
#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
|
||||||
|
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||||
#include "datagen.h" /* RDG_genBuffer */
|
#include "datagen.h" /* RDG_genBuffer */
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#define XXH_STATIC_LINKING_ONLY
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
@ -54,7 +56,6 @@
|
|||||||
#define MB *(1U<<20)
|
#define MB *(1U<<20)
|
||||||
#define GB *(1U<<30)
|
#define GB *(1U<<30)
|
||||||
|
|
||||||
static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */
|
|
||||||
static const U32 FUZ_compressibility_default = 50;
|
static const U32 FUZ_compressibility_default = 50;
|
||||||
static const U32 nbTestsDefault = 30000;
|
static const U32 nbTestsDefault = 30000;
|
||||||
|
|
||||||
@ -78,7 +79,6 @@ static clock_t g_displayClock = 0;
|
|||||||
* Fuzzer functions
|
* Fuzzer functions
|
||||||
*********************************************************/
|
*********************************************************/
|
||||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
|
||||||
|
|
||||||
static clock_t FUZ_clockSpan(clock_t cStart)
|
static clock_t FUZ_clockSpan(clock_t cStart)
|
||||||
{
|
{
|
||||||
@ -86,8 +86,8 @@ static clock_t FUZ_clockSpan(clock_t cStart)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||||
unsigned int FUZ_rand(unsigned int* src)
|
static unsigned FUZ_rand(unsigned* src)
|
||||||
{
|
{
|
||||||
static const U32 prime1 = 2654435761U;
|
static const U32 prime1 = 2654435761U;
|
||||||
static const U32 prime2 = 2246822519U;
|
static const U32 prime2 = 2246822519U;
|
||||||
@ -104,142 +104,131 @@ static unsigned FUZ_highbit32(U32 v32)
|
|||||||
{
|
{
|
||||||
unsigned nbBits = 0;
|
unsigned nbBits = 0;
|
||||||
if (v32==0) return 0;
|
if (v32==0) return 0;
|
||||||
while (v32) {
|
while (v32) v32 >>= 1, nbBits++;
|
||||||
v32 >>= 1;
|
|
||||||
nbBits ++;
|
|
||||||
}
|
|
||||||
return nbBits;
|
return nbBits;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||||
|
#define CHECK(fn) { CHECKTEST(err, fn); }
|
||||||
|
#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; }
|
||||||
static int basicUnitTests(U32 seed, double compressibility)
|
static int basicUnitTests(U32 seed, double compressibility)
|
||||||
{
|
{
|
||||||
|
size_t const CNBuffSize = 5 MB;
|
||||||
|
void* const CNBuffer = malloc(CNBuffSize);
|
||||||
|
void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
|
||||||
|
void* const decodedBuffer = malloc(CNBuffSize);
|
||||||
int testResult = 0;
|
int testResult = 0;
|
||||||
void* CNBuffer;
|
|
||||||
void* compressedBuffer;
|
|
||||||
void* decodedBuffer;
|
|
||||||
U32 randState = seed;
|
|
||||||
size_t result, cSize;
|
|
||||||
U32 testNb=0;
|
U32 testNb=0;
|
||||||
|
size_t cSize;
|
||||||
|
|
||||||
/* Create compressible test buffer */
|
/* Create compressible noise */
|
||||||
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
|
||||||
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
|
|
||||||
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
|
||||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
|
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
|
||||||
DISPLAY("Not enough memory, aborting\n");
|
DISPLAY("Not enough memory, aborting\n");
|
||||||
testResult = 1;
|
testResult = 1;
|
||||||
goto _end;
|
goto _end;
|
||||||
}
|
}
|
||||||
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
|
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
|
||||||
|
|
||||||
/* Basic tests */
|
/* Basic tests */
|
||||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
CNBuffer, CNBuffSize, 1),
|
||||||
cSize = result;
|
cSize=r );
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (r != CNBuffSize) goto _output_error);
|
||||||
if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
{ size_t i;
|
|
||||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||||
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) {
|
{ size_t u;
|
||||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
for (u=0; u<CNBuffSize; u++) {
|
||||||
}
|
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
|
||||||
|
} }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
}
|
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
|
||||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
|
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
|
||||||
if (!ZSTD_isError(result)) goto _output_error;
|
if (!ZSTD_isError(r)) goto _output_error;
|
||||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
|
||||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
|
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
|
||||||
if (!ZSTD_isError(result)) goto _output_error;
|
if (!ZSTD_isError(r)) goto _output_error;
|
||||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* Dictionary and CCtx Duplication tests */
|
/* Dictionary and CCtx Duplication tests */
|
||||||
{ ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
|
||||||
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
|
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
|
||||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
size_t const dictSize = 500;
|
static const size_t dictSize = 551;
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
|
||||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
||||||
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error should be detected */
|
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
|
||||||
{ size_t const initResult = ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2);
|
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
|
||||||
if (ZSTD_isError(initResult)) goto _output_error; }
|
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
|
||||||
if (ZSTD_isError(copyResult)) goto _output_error; }
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
|
||||||
cSize = 0;
|
cSize = 0;
|
||||||
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||||
cSize += result;
|
cSize += r);
|
||||||
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
|
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
cSize += r);
|
||||||
cSize += result;
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
|
||||||
result = ZSTD_decompress_usingDict(dctx,
|
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||||
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
|
decodedBuffer, CNBuffSize,
|
||||||
compressedBuffer, cSize,
|
compressedBuffer, cSize,
|
||||||
CNBuffer, dictSize);
|
CNBuffer, dictSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||||
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
|
||||||
{ size_t const cSizeOrig = cSize;
|
{ size_t const cSizeOrig = cSize;
|
||||||
cSize = 0;
|
cSize = 0;
|
||||||
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||||
cSize += result;
|
cSize += r);
|
||||||
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
|
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
cSize += r);
|
||||||
cSize += result;
|
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> have same size */
|
||||||
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
|
|
||||||
}
|
}
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
|
||||||
result = ZSTD_decompress_usingDict(dctx,
|
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||||
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
|
decodedBuffer, CNBuffSize,
|
||||||
compressedBuffer, cSize,
|
compressedBuffer, cSize,
|
||||||
CNBuffer, dictSize);
|
CNBuffer, dictSize),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||||
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
|
||||||
{ size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3;
|
{ size_t const testSize = CNBuffSize / 3;
|
||||||
{ ZSTD_parameters p;
|
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
|
||||||
p.cParams = ZSTD_getCParams(2, testSize, dictSize);
|
ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
|
||||||
p.fParams.contentSizeFlag = 1;
|
ZSTD_parameters p;
|
||||||
{ size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1);
|
p.cParams = cPar; p.fParams = fPar;
|
||||||
if (ZSTD_isError(initResult)) goto _output_error;
|
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
|
||||||
} }
|
}
|
||||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||||
if (ZSTD_isError(copyResult)) goto _output_error; }
|
|
||||||
cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
|
||||||
if (ZSTD_isError(cSize)) goto _output_error;
|
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||||
|
cSize = r);
|
||||||
{ ZSTD_frameParams fp;
|
{ ZSTD_frameParams fp;
|
||||||
size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize);
|
if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
|
||||||
if (gfpResult!=0) goto _output_error;
|
|
||||||
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
|
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
|
||||||
} }
|
} }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
@ -249,55 +238,113 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
ZSTD_freeDCtx(dctx);
|
ZSTD_freeDCtx(dctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Dictionary and dictBuilder tests */
|
||||||
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
|
size_t dictSize = 16 KB;
|
||||||
|
void* dictBuffer = malloc(dictSize);
|
||||||
|
size_t const totalSampleSize = 1 MB;
|
||||||
|
size_t const sampleUnitSize = 8 KB;
|
||||||
|
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||||
|
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||||
|
|
||||||
|
if (dictBuffer==NULL || samplesSizes==NULL) {
|
||||||
|
free(dictBuffer);
|
||||||
|
free(samplesSizes);
|
||||||
|
goto _output_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
|
||||||
|
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||||
|
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
||||||
|
CNBuffer, samplesSizes, nbSamples);
|
||||||
|
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||||
|
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
||||||
|
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||||
|
CNBuffer, CNBuffSize,
|
||||||
|
dictBuffer, dictSize, 4);
|
||||||
|
if (ZSTD_isError(cSize)) goto _output_error;
|
||||||
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||||
|
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||||
|
decodedBuffer, CNBuffSize,
|
||||||
|
compressedBuffer, cSize,
|
||||||
|
dictBuffer, dictSize),
|
||||||
|
if (r != CNBuffSize) goto _output_error);
|
||||||
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
|
||||||
|
{ ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
|
||||||
|
ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
|
||||||
|
ZSTD_parameters p;
|
||||||
|
p.cParams = cParams; p.fParams = fParams;
|
||||||
|
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||||
|
CNBuffer, CNBuffSize,
|
||||||
|
dictBuffer, dictSize, p);
|
||||||
|
if (ZSTD_isError(cSize)) goto _output_error;
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||||
|
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||||
|
decodedBuffer, CNBuffSize,
|
||||||
|
compressedBuffer, cSize,
|
||||||
|
dictBuffer, dictSize),
|
||||||
|
if (r != CNBuffSize) goto _output_error);
|
||||||
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
free(dictBuffer);
|
||||||
|
free(samplesSizes);
|
||||||
|
}
|
||||||
|
|
||||||
/* Decompression defense tests */
|
/* Decompression defense tests */
|
||||||
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
|
||||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
|
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
|
||||||
if (!ZSTD_isError(result)) goto _output_error;
|
if (!ZSTD_isError(r)) goto _output_error;
|
||||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
|
||||||
((char*)(CNBuffer))[0] = 1;
|
((char*)(CNBuffer))[0] = 1;
|
||||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
|
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
|
||||||
if (!ZSTD_isError(result)) goto _output_error;
|
if (!ZSTD_isError(r)) goto _output_error; }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* block API tests */
|
/* block API tests */
|
||||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
const size_t blockSize = 100 KB;
|
static const size_t blockSize = 100 KB;
|
||||||
const size_t dictSize = 16 KB;
|
static const size_t dictSize = 16 KB;
|
||||||
|
|
||||||
/* basic block compression */
|
/* basic block compression */
|
||||||
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
|
||||||
result = ZSTD_compressBegin(cctx, 5);
|
CHECK( ZSTD_compressBegin(cctx, 5) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
|
||||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
||||||
if (ZSTD_isError(cSize)) goto _output_error;
|
if (ZSTD_isError(cSize)) goto _output_error;
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
|
||||||
result = ZSTD_decompressBegin(dctx);
|
CHECK( ZSTD_decompressBegin(dctx) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
{ CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||||
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
if (r != blockSize) goto _output_error; }
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
|
||||||
if (result != blockSize) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* dictionary block compression */
|
/* dictionary block compression */
|
||||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
|
||||||
result = ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5);
|
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
|
||||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
|
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
|
||||||
if (ZSTD_isError(cSize)) goto _output_error;
|
if (ZSTD_isError(cSize)) goto _output_error;
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
|
||||||
result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize);
|
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
{ CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||||
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
if (r != blockSize) goto _output_error; }
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
|
||||||
if (result != blockSize) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
@ -307,32 +354,29 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
/* long rle test */
|
/* long rle test */
|
||||||
{ size_t sampleSize = 0;
|
{ size_t sampleSize = 0;
|
||||||
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
|
||||||
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
|
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
|
||||||
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
||||||
sampleSize += 256 KB - 1;
|
sampleSize += 256 KB - 1;
|
||||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
|
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
|
||||||
sampleSize += 96 KB;
|
sampleSize += 96 KB;
|
||||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||||
if (ZSTD_isError(cSize)) goto _output_error;
|
if (ZSTD_isError(cSize)) goto _output_error;
|
||||||
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
|
{ CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (regenSize!=sampleSize) goto _output_error; }
|
||||||
if (result!=sampleSize) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
}
|
}
|
||||||
|
|
||||||
/* All zeroes test (#137 verif) */
|
/* All zeroes test (test bug #137) */
|
||||||
#define ZEROESLENGTH 100
|
#define ZEROESLENGTH 100
|
||||||
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
|
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||||
memset(CNBuffer, 0, ZEROESLENGTH);
|
memset(CNBuffer, 0, ZEROESLENGTH);
|
||||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1);
|
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
cSize = r; }
|
||||||
cSize = result;
|
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
|
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||||
result = ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize);
|
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (r != ZEROESLENGTH) goto _output_error; }
|
||||||
if (result != ZEROESLENGTH) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* nbSeq limit test */
|
/* nbSeq limit test */
|
||||||
@ -352,23 +396,21 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}}
|
}}
|
||||||
|
|
||||||
/* randomly fills CNBuffer with prepared 3-bytes sequences */
|
/* randomly fills CNBuffer with prepared 3-bytes sequences */
|
||||||
{ int i; for (i=0; i < _3BYTESTESTLENGTH; ) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
|
{ int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
|
||||||
U32 id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
|
U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
|
||||||
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
|
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
|
||||||
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
|
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
|
||||||
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
|
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
|
||||||
i += 3;
|
|
||||||
} }}
|
} }}
|
||||||
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
|
||||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19);
|
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
CNBuffer, _3BYTESTESTLENGTH, 19) );
|
||||||
cSize = result;
|
cSize = r; }
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
|
||||||
result = ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize);
|
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
|
||||||
if (ZSTD_isError(result)) goto _output_error;
|
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||||
if (result != _3BYTESTESTLENGTH) goto _output_error;
|
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
_end:
|
_end:
|
||||||
@ -388,11 +430,11 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
|||||||
{
|
{
|
||||||
const BYTE* b1 = (const BYTE*)buf1;
|
const BYTE* b1 = (const BYTE*)buf1;
|
||||||
const BYTE* b2 = (const BYTE*)buf2;
|
const BYTE* b2 = (const BYTE*)buf2;
|
||||||
size_t i;
|
size_t u;
|
||||||
for (i=0; i<max; i++) {
|
for (u=0; u<max; u++) {
|
||||||
if (b1[i] != b2[i]) break;
|
if (b1[u] != b2[u]) break;
|
||||||
}
|
}
|
||||||
return i;
|
return u;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -408,6 +450,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
|||||||
return FUZ_rLogLength(seed, logLength);
|
return FUZ_rLogLength(seed, logLength);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#undef CHECK
|
||||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||||
|
|
||||||
@ -415,35 +458,29 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
{
|
{
|
||||||
static const U32 maxSrcLog = 23;
|
static const U32 maxSrcLog = 23;
|
||||||
static const U32 maxSampleLog = 22;
|
static const U32 maxSampleLog = 22;
|
||||||
|
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
|
||||||
|
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
||||||
|
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||||
BYTE* cNoiseBuffer[5];
|
BYTE* cNoiseBuffer[5];
|
||||||
BYTE* srcBuffer;
|
BYTE* srcBuffer; /* jumping pointer */
|
||||||
BYTE* cBuffer;
|
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
||||||
BYTE* dstBuffer;
|
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
BYTE* mirrorBuffer;
|
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
|
||||||
size_t dstBufferSize = (size_t)1<<maxSampleLog;
|
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||||
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
U32 result = 0;
|
U32 result = 0;
|
||||||
U32 testNb = 0;
|
U32 testNb = 0;
|
||||||
U32 coreSeed = seed, lseed = 0;
|
U32 coreSeed = seed, lseed = 0;
|
||||||
ZSTD_CCtx* refCtx;
|
clock_t const startClock = clock();
|
||||||
ZSTD_CCtx* ctx;
|
|
||||||
ZSTD_DCtx* dctx;
|
|
||||||
clock_t startClock = clock();
|
|
||||||
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
|
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
|
||||||
|
|
||||||
/* allocation */
|
/* allocation */
|
||||||
refCtx = ZSTD_createCCtx();
|
|
||||||
ctx = ZSTD_createCCtx();
|
|
||||||
dctx= ZSTD_createDCtx();
|
|
||||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
|
||||||
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
|
|
||||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
|
||||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|
||||||
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
|
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
|
||||||
"Not enough memory, fuzzer tests cancelled");
|
"Not enough memory, fuzzer tests cancelled");
|
||||||
@ -461,9 +498,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
|
|
||||||
/* main test loop */
|
/* main test loop */
|
||||||
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
||||||
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
|
size_t sampleSize, maxTestSize, totalTestSize;
|
||||||
size_t cSize, dSize, totalCSize, totalGenSize;
|
size_t cSize, totalCSize, totalGenSize;
|
||||||
U32 sampleSizeLog, nbChunks, n;
|
|
||||||
XXH64_state_t xxhState;
|
XXH64_state_t xxhState;
|
||||||
U64 crcOrig;
|
U64 crcOrig;
|
||||||
BYTE* sampleBuffer;
|
BYTE* sampleBuffer;
|
||||||
@ -493,18 +529,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* select src segment */
|
/* select src segment */
|
||||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||||
sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog);
|
|
||||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
|
||||||
|
|
||||||
/* create sample buffer (to catch read error with valgrind & sanitizers) */
|
/* create sample buffer (to catch read error with valgrind & sanitizers) */
|
||||||
sampleBuffer = (BYTE*)malloc(sampleSize);
|
sampleBuffer = (BYTE*)malloc(sampleSize);
|
||||||
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
|
CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
|
||||||
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
|
{ size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||||
|
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
|
||||||
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
|
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
|
||||||
|
|
||||||
/* compression tests */
|
/* compression tests */
|
||||||
{ int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1;
|
{ unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
|
||||||
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
|
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
|
||||||
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
||||||
|
|
||||||
@ -518,9 +553,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
|
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
|
||||||
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
||||||
}
|
} }
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/* frame header decompression test */
|
/* frame header decompression test */
|
||||||
{ ZSTD_frameParams dParams;
|
{ ZSTD_frameParams dParams;
|
||||||
@ -531,7 +564,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
|
|
||||||
/* successful decompression test */
|
/* successful decompression test */
|
||||||
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
|
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
|
||||||
dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
|
size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
|
||||||
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
|
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
|
||||||
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
|
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
|
||||||
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
|
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
|
||||||
@ -542,7 +575,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
/* truncated src decompression test */
|
/* truncated src decompression test */
|
||||||
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||||
size_t const tooSmallSize = cSize - missing;
|
size_t const tooSmallSize = cSize - missing;
|
||||||
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
|
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
|
||||||
CHECK(cBufferTooSmall == NULL, "not enough memory !");
|
CHECK(cBufferTooSmall == NULL, "not enough memory !");
|
||||||
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
|
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
|
||||||
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
|
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
|
||||||
@ -575,13 +608,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
}
|
}
|
||||||
if (pos <= cSize) break;
|
if (pos <= cSize) break;
|
||||||
/* add noise */
|
/* add noise */
|
||||||
{ U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
|
{ U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
|
||||||
size_t mask, noiseStart, noiseLength;
|
U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
|
||||||
if (nbBits>0) nbBits--;
|
size_t const mask = (1<<nbBits) - 1;
|
||||||
mask = (1<<nbBits) - 1;
|
size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
|
||||||
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
|
size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
|
||||||
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
|
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
|
||||||
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
|
|
||||||
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
|
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
|
||||||
pos += noiseLength;
|
pos += noiseLength;
|
||||||
} } }
|
} } }
|
||||||
@ -605,35 +637,45 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||||
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
|
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
|
||||||
|
|
||||||
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
|
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
|
||||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
|
||||||
dict = srcBuffer + sampleStart;
|
|
||||||
dictSize = sampleSize;
|
|
||||||
|
|
||||||
{ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
if (FUZ_rand(&lseed) & 0xF) {
|
||||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
|
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
||||||
|
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
|
||||||
|
} else {
|
||||||
|
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||||
|
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||||
|
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
|
||||||
|
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||||
|
ZSTD_parameters p;
|
||||||
|
size_t errorCode;
|
||||||
|
p.cParams = cPar; p.fParams = fpar;
|
||||||
|
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
|
||||||
|
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
|
||||||
|
}
|
||||||
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
|
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
|
||||||
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
|
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
|
||||||
}
|
}
|
||||||
XXH64_reset(&xxhState, 0);
|
XXH64_reset(&xxhState, 0);
|
||||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||||
|
U32 n;
|
||||||
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
||||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||||
sampleSize = (size_t)1 << sampleSizeLog;
|
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
|
||||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
|
||||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
|
||||||
|
|
||||||
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) break; /* avoid invalid dstBufferTooSmall */
|
if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
|
||||||
if (totalTestSize+sampleSize > maxTestSize) break;
|
if (totalTestSize+segmentSize > maxTestSize) break;
|
||||||
|
|
||||||
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
|
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
|
||||||
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
|
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
|
||||||
cSize += compressResult;
|
cSize += compressResult;
|
||||||
}
|
}
|
||||||
XXH64_update(&xxhState, srcBuffer+sampleStart, sampleSize);
|
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
|
||||||
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
|
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
|
||||||
totalTestSize += sampleSize;
|
totalTestSize += segmentSize;
|
||||||
}
|
} }
|
||||||
|
|
||||||
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
|
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
|
||||||
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
||||||
cSize += flushResult;
|
cSize += flushResult;
|
||||||
@ -641,8 +683,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
crcOrig = XXH64_digest(&xxhState);
|
crcOrig = XXH64_digest(&xxhState);
|
||||||
|
|
||||||
/* streaming decompression test */
|
/* streaming decompression test */
|
||||||
|
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
|
||||||
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||||
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); }
|
CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
|
||||||
totalCSize = 0;
|
totalCSize = 0;
|
||||||
totalGenSize = 0;
|
totalGenSize = 0;
|
||||||
while (totalCSize < cSize) {
|
while (totalCSize < cSize) {
|
||||||
@ -653,12 +696,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
totalCSize += inSize;
|
totalCSize += inSize;
|
||||||
}
|
}
|
||||||
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
|
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
|
||||||
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
|
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
||||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||||
if (crcDest!=crcOrig) {
|
if (crcDest!=crcOrig) {
|
||||||
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
||||||
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
||||||
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
||||||
} }
|
} }
|
||||||
} /* for ( ; (testNb <= nbTests) */
|
} /* for ( ; (testNb <= nbTests) */
|
||||||
|
@ -48,7 +48,8 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "zstd_static.h"
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
||||||
|
#include "zstd.h"
|
||||||
#include "datagen.h"
|
#include "datagen.h"
|
||||||
#include "xxhash.h"
|
#include "xxhash.h"
|
||||||
|
|
||||||
@ -600,22 +601,22 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
|
static ZSTD_compressionParameters randomParams(void)
|
||||||
{
|
{
|
||||||
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
|
ZSTD_compressionParameters p;
|
||||||
U32 validated = 0;
|
U32 validated = 0;
|
||||||
if (!chance)
|
|
||||||
while (!validated) {
|
while (!validated) {
|
||||||
/* totally random entry */
|
/* totally random entry */
|
||||||
p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
||||||
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||||
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||||
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||||
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||||
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||||
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||||
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
|
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||||
}
|
}
|
||||||
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_selectRandomStart(
|
static void BMK_selectRandomStart(
|
||||||
@ -623,12 +624,10 @@ static void BMK_selectRandomStart(
|
|||||||
const void* srcBuffer, size_t srcSize,
|
const void* srcBuffer, size_t srcSize,
|
||||||
ZSTD_CCtx* ctx)
|
ZSTD_CCtx* ctx)
|
||||||
{
|
{
|
||||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||||
/* totally random entry */
|
/* totally random entry */
|
||||||
ZSTD_compressionParameters p;
|
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
||||||
potentialRandomParams(&p, 1);
|
|
||||||
ZSTD_adjustCParams(&p, srcSize, 0);
|
|
||||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -649,7 +648,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
|||||||
|
|
||||||
if (g_singleRun) {
|
if (g_singleRun) {
|
||||||
BMK_result_t testResult;
|
BMK_result_t testResult;
|
||||||
ZSTD_adjustCParams(&g_params, srcSize, 0);
|
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
||||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||||
DISPLAY("\n");
|
DISPLAY("\n");
|
||||||
return;
|
return;
|
||||||
@ -861,7 +860,7 @@ int optimizeForSize(char* inFileName)
|
|||||||
do {
|
do {
|
||||||
params = winner.params;
|
params = winner.params;
|
||||||
paramVariation(¶ms);
|
paramVariation(¶ms);
|
||||||
potentialRandomParams(¶ms, 16);
|
if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams();
|
||||||
|
|
||||||
/* exclude faster if already played set of params */
|
/* exclude faster if already played set of params */
|
||||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
||||||
|
@ -15,22 +15,30 @@ roundTripTest() {
|
|||||||
|
|
||||||
rm -f tmp1 tmp2
|
rm -f tmp1 tmp2
|
||||||
$ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
|
$ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
|
||||||
./datagen $1 $p | md5sum > tmp1
|
./datagen $1 $p | $MD5SUM > tmp1
|
||||||
./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2
|
./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2
|
||||||
diff -q tmp1 tmp2
|
diff -q tmp1 tmp2
|
||||||
}
|
}
|
||||||
|
|
||||||
isWindows=false
|
isWindows=false
|
||||||
ECHO="echo"
|
ECHO="echo"
|
||||||
if [[ "$OS" == "Windows"* ]]; then
|
case "$OS" in
|
||||||
|
Windows*)
|
||||||
isWindows=true
|
isWindows=true
|
||||||
ECHO="echo -e"
|
ECHO="echo -e"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
MD5SUM="md5sum"
|
||||||
|
if [ "$TRAVIS_OS_NAME" = "osx" ]; then
|
||||||
|
MD5SUM="md5 -r"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$ECHO "\nStarting playTests.sh isWindows=$isWindows"
|
$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"
|
||||||
|
|
||||||
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
||||||
|
|
||||||
|
file $ZSTD
|
||||||
$ECHO "\n**** simple tests **** "
|
$ECHO "\n**** simple tests **** "
|
||||||
|
|
||||||
./datagen > tmp
|
./datagen > tmp
|
||||||
@ -114,19 +122,32 @@ $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
|
|||||||
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
|
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
|
||||||
ls -ls tmpSparse*
|
ls -ls tmpSparse*
|
||||||
diff tmpSparse2M tmpSparseRegenerated
|
diff tmpSparse2M tmpSparseRegenerated
|
||||||
# rm tmpSparse*
|
rm tmpSparse*
|
||||||
|
|
||||||
|
|
||||||
$ECHO "\n**** dictionary tests **** "
|
$ECHO "\n**** dictionary tests **** "
|
||||||
|
|
||||||
./datagen > tmpDict
|
./datagen > tmpDict
|
||||||
./datagen -g1M | md5sum > tmp1
|
./datagen -g1M | $MD5SUM > tmp1
|
||||||
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
|
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
|
||||||
diff -q tmp1 tmp2
|
diff -q tmp1 tmp2
|
||||||
$ZSTD --train *.c *.h -o tmpDict
|
$ECHO "Create first dictionary"
|
||||||
$ZSTD xxhash.c -D tmpDict -of tmp
|
$ZSTD --train *.c -o tmpDict
|
||||||
$ZSTD -d tmp -D tmpDict -of result
|
cp zstdcli.c tmp
|
||||||
diff xxhash.c result
|
$ZSTD -f tmp -D tmpDict
|
||||||
|
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||||
|
diff zstdcli.c result
|
||||||
|
$ECHO "Create second (different) dictionary"
|
||||||
|
$ZSTD --train *.c *.h -o tmpDictC
|
||||||
|
$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
|
||||||
|
$ECHO "Create dictionary with short dictID"
|
||||||
|
$ZSTD --train *.c --dictID 1 -o tmpDict1
|
||||||
|
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||||
|
$ECHO "Compress without dictID"
|
||||||
|
$ZSTD -f tmp -D tmpDict1 --no-dictID
|
||||||
|
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||||
|
diff zstdcli.c result
|
||||||
|
rm tmp*
|
||||||
|
|
||||||
|
|
||||||
$ECHO "\n**** multiple files tests **** "
|
$ECHO "\n**** multiple files tests **** "
|
||||||
@ -210,4 +231,3 @@ roundTripTest -g99000000 -P99 20
|
|||||||
roundTripTest -g6000000000 -P99 1
|
roundTripTest -g6000000000 -P99 1
|
||||||
|
|
||||||
rm tmp*
|
rm tmp*
|
||||||
|
|
@ -1,157 +0,0 @@
|
|||||||
@echo off
|
|
||||||
ECHO Start playTests.bat
|
|
||||||
ECHO ZSTD^: %ZSTD%
|
|
||||||
ECHO DATAGEN^: %DATAGEN%
|
|
||||||
if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1
|
|
||||||
if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1
|
|
||||||
|
|
||||||
SET ROUNDTRIPTEST=tests\roundTripTest.bat
|
|
||||||
|
|
||||||
echo. && echo **** simple tests ****
|
|
||||||
%DATAGEN% > tmp
|
|
||||||
%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst
|
|
||||||
%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp)
|
|
||||||
echo test : too large compression level (must fail)
|
|
||||||
%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1)
|
|
||||||
echo test : compress to stdout
|
|
||||||
%ZSTD% tmp -c > tmpCompressed
|
|
||||||
%ZSTD% tmp --stdout > tmpCompressed && REM long command format
|
|
||||||
echo test : null-length file roundtrip
|
|
||||||
echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1)
|
|
||||||
echo test : decompress file with wrong suffix (must fail)
|
|
||||||
%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1)
|
|
||||||
%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout
|
|
||||||
%ZSTD% --decompress tmpCompressed -c > tmpResult
|
|
||||||
%ZSTD% --decompress tmpCompressed --stdout > tmpResult
|
|
||||||
REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout
|
|
||||||
REM %ZSTD% -d - < tmp.zst > NUL
|
|
||||||
%ZSTD% -dc < tmp.zst > NUL
|
|
||||||
%ZSTD% -dc - < tmp.zst > NUL
|
|
||||||
%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1)
|
|
||||||
%ZSTD% -q -f tmp
|
|
||||||
%ZSTD% -q --force tmp
|
|
||||||
%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1)
|
|
||||||
|
|
||||||
echo. && echo **** Pass-Through mode ****
|
|
||||||
echo "Hello world !" | %ZSTD% -df
|
|
||||||
echo "Hello world !" | %ZSTD% -dcf
|
|
||||||
|
|
||||||
|
|
||||||
echo. && echo **** frame concatenation ****
|
|
||||||
|
|
||||||
echo hello > hello.tmp
|
|
||||||
echo world! > world.tmp
|
|
||||||
cat hello.tmp world.tmp > helloworld.tmp
|
|
||||||
%ZSTD% -c hello.tmp > hello.zstd
|
|
||||||
%ZSTD% -c world.tmp > world.zstd
|
|
||||||
cat hello.zstd world.zstd > helloworld.zstd
|
|
||||||
%ZSTD% -dc helloworld.zstd > result.tmp
|
|
||||||
cat result.tmp
|
|
||||||
fc /b helloworld.tmp result.tmp
|
|
||||||
rm *.tmp *.zstd
|
|
||||||
|
|
||||||
echo frame concatenation tests completed
|
|
||||||
|
|
||||||
|
|
||||||
REM echo. && echo **** flush write error test ****
|
|
||||||
|
|
||||||
REM echo echo foo ^| %ZSTD% ^> v:\full
|
|
||||||
REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1)
|
|
||||||
REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full"
|
|
||||||
REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1)
|
|
||||||
|
|
||||||
|
|
||||||
echo. && echo **** dictionary tests ****
|
|
||||||
|
|
||||||
%DATAGEN% > tmpDict
|
|
||||||
%DATAGEN% -g1M | md5sum > tmp1
|
|
||||||
%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2
|
|
||||||
fc tmp1 tmp2
|
|
||||||
%ZSTD% --train *.c *.h -o tmpDict
|
|
||||||
%ZSTD% xxhash.c -D tmpDict -of tmp
|
|
||||||
%ZSTD% -d tmp -D tmpDict -of result
|
|
||||||
fc xxhash.c result
|
|
||||||
|
|
||||||
|
|
||||||
echo. && echo **** multiple files tests ****
|
|
||||||
|
|
||||||
%DATAGEN% -s1 > tmp1 2> NUL
|
|
||||||
%DATAGEN% -s2 -g100K > tmp2 2> NUL
|
|
||||||
%DATAGEN% -s3 -g1M > tmp3 2> NUL
|
|
||||||
%ZSTD% -f tmp*
|
|
||||||
echo compress tmp* :
|
|
||||||
ls -ls tmp*
|
|
||||||
rm tmp1 tmp2 tmp3
|
|
||||||
echo decompress tmp* :
|
|
||||||
%ZSTD% -df *.zst
|
|
||||||
ls -ls tmp*
|
|
||||||
|
|
||||||
echo compress tmp* into stdout ^> tmpall :
|
|
||||||
%ZSTD% -c tmp1 tmp2 tmp3 > tmpall
|
|
||||||
ls -ls tmp*
|
|
||||||
echo decompress tmpall* into stdout ^> tmpdec :
|
|
||||||
cp tmpall tmpall2
|
|
||||||
%ZSTD% -dc tmpall* > tmpdec
|
|
||||||
ls -ls tmp*
|
|
||||||
echo compress multiple files including a missing one (notHere) :
|
|
||||||
%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1)
|
|
||||||
|
|
||||||
|
|
||||||
echo. && echo **** integrity tests ****
|
|
||||||
echo test one file (tmp1.zst)
|
|
||||||
%ZSTD% -t tmp1.zst
|
|
||||||
%ZSTD% --test tmp1.zst
|
|
||||||
echo test multiple files (*.zst)
|
|
||||||
%ZSTD% -t *.zst
|
|
||||||
echo test good and bad files (*)
|
|
||||||
%ZSTD% -t * && (echo bad files not detected! && exit /b 1)
|
|
||||||
|
|
||||||
|
|
||||||
echo. && echo **** zstd round-trip tests ****
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST%
|
|
||||||
CALL %ROUNDTRIPTEST% -g15K && REM TableID==3
|
|
||||||
CALL %ROUNDTRIPTEST% -g127K && REM TableID==2
|
|
||||||
CALL %ROUNDTRIPTEST% -g255K && REM TableID==1
|
|
||||||
CALL %ROUNDTRIPTEST% -g513K && REM TableID==0
|
|
||||||
CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain
|
|
||||||
CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2
|
|
||||||
CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt
|
|
||||||
|
|
||||||
rm tmp*
|
|
||||||
echo Param = %1
|
|
||||||
if NOT "%1"=="--test-large-data" (
|
|
||||||
echo skipping large data tests
|
|
||||||
exit /b 0
|
|
||||||
)
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g270000000 1
|
|
||||||
CALL %ROUNDTRIPTEST% -g270000000 2
|
|
||||||
CALL %ROUNDTRIPTEST% -g270000000 3
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g140000000 -P60 4
|
|
||||||
CALL %ROUNDTRIPTEST% -g140000000 -P60 5
|
|
||||||
CALL %ROUNDTRIPTEST% -g140000000 -P60 6
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g70000000 -P70 7
|
|
||||||
CALL %ROUNDTRIPTEST% -g70000000 -P70 8
|
|
||||||
CALL %ROUNDTRIPTEST% -g70000000 -P70 9
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g35000000 -P75 10
|
|
||||||
CALL %ROUNDTRIPTEST% -g35000000 -P75 11
|
|
||||||
CALL %ROUNDTRIPTEST% -g35000000 -P75 12
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g18000000 -P80 13
|
|
||||||
CALL %ROUNDTRIPTEST% -g18000000 -P80 14
|
|
||||||
CALL %ROUNDTRIPTEST% -g18000000 -P80 15
|
|
||||||
CALL %ROUNDTRIPTEST% -g18000000 -P80 16
|
|
||||||
CALL %ROUNDTRIPTEST% -g18000000 -P80 17
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g50000000 -P94 18
|
|
||||||
CALL %ROUNDTRIPTEST% -g50000000 -P94 19
|
|
||||||
|
|
||||||
CALL %ROUNDTRIPTEST% -g99000000 -P99 20
|
|
||||||
CALL %ROUNDTRIPTEST% -g6000000000 -P99 1
|
|
||||||
|
|
||||||
rm tmp*
|
|
||||||
exit /b 0
|
|
@ -1,8 +0,0 @@
|
|||||||
@echo off
|
|
||||||
if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2)
|
|
||||||
rm -f tmp1 tmp2
|
|
||||||
echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d
|
|
||||||
%DATAGEN% %1 %P% | md5sum > tmp1
|
|
||||||
%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2
|
|
||||||
fc tmp1 tmp2
|
|
||||||
EXIT /B %ERRORLEVEL%
|
|
@ -40,12 +40,13 @@
|
|||||||
#include <sys/timeb.h> /* timeb */
|
#include <sys/timeb.h> /* timeb */
|
||||||
#include <string.h> /* strcmp */
|
#include <string.h> /* strcmp */
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
#include "zbuff.h"
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||||
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
|
#include "zstd.h" /* ZSTD_compressBound */
|
||||||
#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */
|
#define ZBUFF_STATIC_LINKING_ONLY
|
||||||
|
#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
|
||||||
#include "datagen.h" /* RDG_genBuffer */
|
#include "datagen.h" /* RDG_genBuffer */
|
||||||
#define XXH_STATIC_LINKING_ONLY
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
#include "xxhash.h" /* XXH64 */
|
#include "xxhash.h" /* XXH64_* */
|
||||||
|
|
||||||
|
|
||||||
/*-************************************
|
/*-************************************
|
||||||
@ -97,7 +98,7 @@ static U32 FUZ_GetMilliStart(void)
|
|||||||
|
|
||||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||||
{
|
{
|
||||||
U32 nCurrent = FUZ_GetMilliStart();
|
U32 const nCurrent = FUZ_GetMilliStart();
|
||||||
U32 nSpan = nCurrent - nTimeStart;
|
U32 nSpan = nCurrent - nTimeStart;
|
||||||
if (nTimeStart > nCurrent)
|
if (nTimeStart > nCurrent)
|
||||||
nSpan += 0x100000 * 1000;
|
nSpan += 0x100000 * 1000;
|
||||||
@ -129,16 +130,18 @@ static unsigned FUZ_highbit32(U32 v32)
|
|||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
void* ZBUFF_allocFunction(size_t size)
|
static void* ZBUFF_allocFunction(void* opaque, size_t size)
|
||||||
{
|
{
|
||||||
void* address = malloc(size);
|
void* address = malloc(size);
|
||||||
/* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */
|
(void)opaque;
|
||||||
|
/* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||||
return address;
|
return address;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ZBUFF_freeFunction(void* address)
|
static void ZBUFF_freeFunction(void* opaque, void* address)
|
||||||
{
|
{
|
||||||
/* if (address) DISPLAYLEVEL(4, "free %p \n", address); */
|
(void)opaque;
|
||||||
|
/* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */
|
||||||
free(address);
|
free(address);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -147,11 +150,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||||||
int testResult = 0;
|
int testResult = 0;
|
||||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||||
void* CNBuffer = malloc(CNBufferSize);
|
void* CNBuffer = malloc(CNBufferSize);
|
||||||
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
size_t const skippableFrameSize = 11;
|
||||||
|
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||||
void* compressedBuffer = malloc(compressedBufferSize);
|
void* compressedBuffer = malloc(compressedBufferSize);
|
||||||
size_t const decodedBufferSize = CNBufferSize;
|
size_t const decodedBufferSize = CNBufferSize;
|
||||||
void* decodedBuffer = malloc(decodedBufferSize);
|
void* decodedBuffer = malloc(decodedBufferSize);
|
||||||
size_t result, cSize, readSize, genSize;
|
size_t cSize, readSize, readSkipSize, genSize;
|
||||||
U32 testNb=0;
|
U32 testNb=0;
|
||||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
|
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
|
||||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
|
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
|
||||||
@ -163,45 +167,61 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||||||
}
|
}
|
||||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
||||||
|
|
||||||
|
/* generate skippable frame */
|
||||||
|
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||||
|
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||||
|
cSize = skippableFrameSize + 8;
|
||||||
|
|
||||||
/* Basic compression test */
|
/* Basic compression test */
|
||||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||||
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
||||||
readSize = CNBufferSize;
|
readSize = CNBufferSize;
|
||||||
genSize = compressedBufferSize;
|
genSize = compressedBufferSize;
|
||||||
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
|
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
|
||||||
if (ZBUFF_isError(result)) goto _output_error;
|
if (ZBUFF_isError(r)) goto _output_error; }
|
||||||
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
||||||
cSize = genSize;
|
cSize += genSize;
|
||||||
genSize = compressedBufferSize - cSize;
|
genSize = compressedBufferSize - cSize;
|
||||||
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||||
if (result != 0) goto _output_error; /* error, or some data not flushed */
|
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
|
||||||
cSize += genSize;
|
cSize += genSize;
|
||||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||||
|
|
||||||
|
/* skippable frame test */
|
||||||
|
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
|
||||||
|
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||||
|
readSkipSize = cSize;
|
||||||
|
genSize = CNBufferSize;
|
||||||
|
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||||
|
if (r != 0) goto _output_error; }
|
||||||
|
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
|
||||||
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* Basic decompression test */
|
/* Basic decompression test */
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||||
readSize = cSize;
|
readSize = cSize - readSkipSize;
|
||||||
genSize = CNBufferSize;
|
genSize = CNBufferSize;
|
||||||
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
|
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
|
||||||
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
|
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* check regenerated data is byte exact */
|
/* check regenerated data is byte exact */
|
||||||
{ size_t i;
|
|
||||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||||
|
{ size_t i;
|
||||||
for (i=0; i<CNBufferSize; i++) {
|
for (i=0; i<CNBufferSize; i++) {
|
||||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||||
}
|
} }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
}
|
|
||||||
|
|
||||||
/* Byte-by-byte decompression test */
|
/* Byte-by-byte decompression test */
|
||||||
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
||||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
{ size_t r, pIn=0, pOut=0;
|
||||||
{ size_t r = 1, pIn=0, pOut=0;
|
do
|
||||||
|
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||||
|
r = 1;
|
||||||
while (r) {
|
while (r) {
|
||||||
size_t inS = 1;
|
size_t inS = 1;
|
||||||
size_t outS = 1;
|
size_t outS = 1;
|
||||||
@ -211,19 +231,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||||||
}
|
}
|
||||||
readSize = pIn;
|
readSize = pIn;
|
||||||
genSize = pOut;
|
genSize = pOut;
|
||||||
|
} while (genSize==0);
|
||||||
}
|
}
|
||||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
/* check regenerated data is byte exact */
|
/* check regenerated data is byte exact */
|
||||||
{ size_t i;
|
|
||||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||||
|
{ size_t i;
|
||||||
for (i=0; i<CNBufferSize; i++) {
|
for (i=0; i<CNBufferSize; i++) {
|
||||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||||
}
|
} }
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
}
|
|
||||||
|
|
||||||
_end:
|
_end:
|
||||||
ZBUFF_freeCCtx(zc);
|
ZBUFF_freeCCtx(zc);
|
||||||
@ -332,8 +352,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
|||||||
FUZ_rand(&coreSeed);
|
FUZ_rand(&coreSeed);
|
||||||
lseed = coreSeed ^ prime1;
|
lseed = coreSeed ^ prime1;
|
||||||
|
|
||||||
/* state total reset */
|
/* states full reset (unsynchronized) */
|
||||||
/* some problems only happen when states are re-used in a specific order */
|
/* some issues only happen when reusing states in a specific sequence of parameters */
|
||||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
||||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
||||||
|
|
||||||
@ -356,15 +376,21 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
|||||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||||
/* random dictionary selection */
|
|
||||||
{ size_t dictStart;
|
|
||||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||||
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
/* random dictionary selection */
|
||||||
|
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||||
dict = srcBuffer + dictStart;
|
dict = srcBuffer + dictStart;
|
||||||
}
|
}
|
||||||
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
|
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||||
|
U32 const checksum = FUZ_rand(&lseed) & 1;
|
||||||
|
U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||||
|
ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
|
||||||
|
ZSTD_parameters params;
|
||||||
|
params.cParams = cPar;
|
||||||
|
params.fParams = fPar;
|
||||||
|
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
|
||||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||||
} }
|
} } }
|
||||||
|
|
||||||
/* multi-segments compression test */
|
/* multi-segments compression test */
|
||||||
XXH64_reset(&xxhState, 0);
|
XXH64_reset(&xxhState, 0);
|
||||||
@ -498,8 +524,8 @@ int main(int argc, const char** argv)
|
|||||||
int result=0;
|
int result=0;
|
||||||
U32 mainPause = 0;
|
U32 mainPause = 0;
|
||||||
const char* programName = argv[0];
|
const char* programName = argv[0];
|
||||||
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction };
|
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL };
|
||||||
ZSTD_customMem customNULL = { NULL, NULL };
|
ZSTD_customMem customNULL = { NULL, NULL, NULL };
|
||||||
|
|
||||||
/* Check command line */
|
/* Check command line */
|
||||||
for(argNb=1; argNb<argc; argNb++) {
|
for(argNb=1; argNb<argc; argNb++) {
|
||||||
@ -600,9 +626,10 @@ int main(int argc, const char** argv)
|
|||||||
|
|
||||||
if (testNb==0) {
|
if (testNb==0) {
|
||||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||||
if (!result)
|
if (!result) {
|
||||||
|
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
|
||||||
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
||||||
}
|
} }
|
||||||
|
|
||||||
if (!result)
|
if (!result)
|
||||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||||
|
@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
|||||||
use `file` as Dictionary to compress or decompress FILE(s)
|
use `file` as Dictionary to compress or decompress FILE(s)
|
||||||
.TP
|
.TP
|
||||||
.B \-o file
|
.B \-o file
|
||||||
save result into `file` (only possible with a single input FILE)
|
save result into `file` (only possible with a single INPUT-FILE)
|
||||||
.TP
|
.TP
|
||||||
.BR \-f ", " --force
|
.BR \-f ", " --force
|
||||||
overwrite output without prompting
|
overwrite output without prompting
|
||||||
@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
|||||||
.TP
|
.TP
|
||||||
.BR \-c ", " --stdout
|
.BR \-c ", " --stdout
|
||||||
force write to standard output, even if it is the console
|
force write to standard output, even if it is the console
|
||||||
|
.TP
|
||||||
|
.BR \-C ", " --check
|
||||||
|
add integrity check computed from uncompressed data
|
||||||
|
.TP
|
||||||
|
.BR \-t ", " --test
|
||||||
|
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
|
||||||
|
No files are created or removed.
|
||||||
|
|
||||||
.SH DICTIONARY
|
.SH DICTIONARY
|
||||||
.PP
|
.PP
|
||||||
@ -92,6 +99,14 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
|
|||||||
.B \--maxdict #
|
.B \--maxdict #
|
||||||
limit dictionary to specified size (default : 112640)
|
limit dictionary to specified size (default : 112640)
|
||||||
.TP
|
.TP
|
||||||
|
.B \--dictID #
|
||||||
|
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
|
||||||
|
By default, zstd will create a 4-bytes random number ID.
|
||||||
|
It's possible to give a precise number instead.
|
||||||
|
Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
|
||||||
|
and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
|
||||||
|
However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
|
||||||
|
.TP
|
||||||
.B \-s#
|
.B \-s#
|
||||||
dictionary selectivity level (default: 9)
|
dictionary selectivity level (default: 9)
|
||||||
the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
|
the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
|
||||||
|
@ -38,10 +38,11 @@
|
|||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
|
# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
|
||||||
#endif
|
#endif
|
||||||
#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */
|
|
||||||
#ifndef ZSTD_NODICT
|
#ifndef ZSTD_NODICT
|
||||||
# include "dibio.h"
|
# include "dibio.h"
|
||||||
#endif
|
#endif
|
||||||
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||||
|
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -52,7 +53,7 @@
|
|||||||
# include <io.h> /* _isatty */
|
# include <io.h> /* _isatty */
|
||||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||||
#else
|
#else
|
||||||
#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
|
#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
|
||||||
# include <unistd.h> /* isatty */
|
# include <unistd.h> /* isatty */
|
||||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||||
#else
|
#else
|
||||||
@ -125,7 +126,6 @@ static int usage_advanced(const char* programName)
|
|||||||
DISPLAY( "\n");
|
DISPLAY( "\n");
|
||||||
DISPLAY( "Advanced arguments :\n");
|
DISPLAY( "Advanced arguments :\n");
|
||||||
DISPLAY( " -V : display Version number and exit\n");
|
DISPLAY( " -V : display Version number and exit\n");
|
||||||
DISPLAY( " -t : test compressed file integrity \n");
|
|
||||||
DISPLAY( " -v : verbose mode\n");
|
DISPLAY( " -v : verbose mode\n");
|
||||||
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
||||||
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
||||||
@ -134,8 +134,13 @@ static int usage_advanced(const char* programName)
|
|||||||
#endif
|
#endif
|
||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
|
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
|
||||||
|
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
|
||||||
|
DISPLAY( "--check : enable integrity check\n");
|
||||||
#endif
|
#endif
|
||||||
|
#ifndef ZSTD_NODECOMPRESS
|
||||||
|
DISPLAY( "--test : test compressed file integrity \n");
|
||||||
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
||||||
|
#endif
|
||||||
#ifndef ZSTD_NODICT
|
#ifndef ZSTD_NODICT
|
||||||
DISPLAY( "\n");
|
DISPLAY( "\n");
|
||||||
DISPLAY( "Dictionary builder :\n");
|
DISPLAY( "Dictionary builder :\n");
|
||||||
@ -143,6 +148,7 @@ static int usage_advanced(const char* programName)
|
|||||||
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||||
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
|
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
|
||||||
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
|
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
|
||||||
|
DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n");
|
||||||
#endif
|
#endif
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
DISPLAY( "\n");
|
DISPLAY( "\n");
|
||||||
@ -171,6 +177,18 @@ static void waitEnter(void)
|
|||||||
(void)unused;
|
(void)unused;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*! readU32FromChar() :
|
||||||
|
@return : unsigned integer value reach from input in `char` format
|
||||||
|
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||||
|
Note : this function can overflow if result > MAX_UNIT */
|
||||||
|
static unsigned readU32FromChar(const char** stringPtr)
|
||||||
|
{
|
||||||
|
unsigned result = 0;
|
||||||
|
while ((**stringPtr >='0') && (**stringPtr <='9'))
|
||||||
|
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
||||||
|
|
||||||
@ -185,7 +203,8 @@ int main(int argCount, const char** argv)
|
|||||||
operationResult=0,
|
operationResult=0,
|
||||||
dictBuild=0,
|
dictBuild=0,
|
||||||
nextArgumentIsOutFileName=0,
|
nextArgumentIsOutFileName=0,
|
||||||
nextArgumentIsMaxDict=0;
|
nextArgumentIsMaxDict=0,
|
||||||
|
nextArgumentIsDictID=0;
|
||||||
unsigned cLevel = 1;
|
unsigned cLevel = 1;
|
||||||
unsigned cLevelLast = 1;
|
unsigned cLevelLast = 1;
|
||||||
unsigned recursive = 0;
|
unsigned recursive = 0;
|
||||||
@ -196,6 +215,7 @@ int main(int argCount, const char** argv)
|
|||||||
const char* dictFileName = NULL;
|
const char* dictFileName = NULL;
|
||||||
char* dynNameSpace = NULL;
|
char* dynNameSpace = NULL;
|
||||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||||
|
unsigned dictID = 0;
|
||||||
unsigned dictCLevel = g_defaultDictCLevel;
|
unsigned dictCLevel = g_defaultDictCLevel;
|
||||||
unsigned dictSelect = g_defaultSelectivityLevel;
|
unsigned dictSelect = g_defaultSelectivityLevel;
|
||||||
#ifdef UTIL_HAS_CREATEFILELIST
|
#ifdef UTIL_HAS_CREATEFILELIST
|
||||||
@ -205,7 +225,8 @@ int main(int argCount, const char** argv)
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
(void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
|
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
|
||||||
|
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
|
||||||
(void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
|
(void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
|
||||||
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
|
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
|
||||||
filenameTable[0] = stdinmark;
|
filenameTable[0] = stdinmark;
|
||||||
@ -233,13 +254,16 @@ int main(int argCount, const char** argv)
|
|||||||
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
|
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
|
||||||
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
||||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
|
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
|
||||||
|
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
||||||
|
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||||
|
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||||
|
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||||
|
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
||||||
if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
|
if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
|
||||||
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
|
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
|
||||||
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
|
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
|
||||||
|
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
|
||||||
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
|
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
|
||||||
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
|
||||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
|
||||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
|
||||||
|
|
||||||
/* '-' means stdin/stdout */
|
/* '-' means stdin/stdout */
|
||||||
if (!strcmp(argument, "-")){
|
if (!strcmp(argument, "-")){
|
||||||
@ -254,12 +278,7 @@ int main(int argCount, const char** argv)
|
|||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
/* compression Level */
|
/* compression Level */
|
||||||
if ((*argument>='0') && (*argument<='9')) {
|
if ((*argument>='0') && (*argument<='9')) {
|
||||||
cLevel = 0;
|
cLevel = readU32FromChar(&argument);
|
||||||
while ((*argument >= '0') && (*argument <= '9')) {
|
|
||||||
cLevel *= 10;
|
|
||||||
cLevel += *argument - '0';
|
|
||||||
argument++;
|
|
||||||
}
|
|
||||||
dictCLevel = cLevel;
|
dictCLevel = cLevel;
|
||||||
if (dictCLevel > ZSTD_maxCLevel())
|
if (dictCLevel > ZSTD_maxCLevel())
|
||||||
CLEAN_RETURN(badusage(programName));
|
CLEAN_RETURN(badusage(programName));
|
||||||
@ -295,8 +314,11 @@ int main(int argCount, const char** argv)
|
|||||||
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
|
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
|
||||||
case 'k': argument++; break;
|
case 'k': argument++; break;
|
||||||
|
|
||||||
|
/* Checksum */
|
||||||
|
case 'C': argument++; FIO_setChecksumFlag(2); break;
|
||||||
|
|
||||||
/* test compressed file */
|
/* test compressed file */
|
||||||
case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break;
|
case 't': decode=1; outFileName=nulmark; argument++; break;
|
||||||
|
|
||||||
/* dictionary name */
|
/* dictionary name */
|
||||||
case 'o': nextArgumentIsOutFileName=1; argument++; break;
|
case 'o': nextArgumentIsOutFileName=1; argument++; break;
|
||||||
@ -312,19 +334,13 @@ int main(int argCount, const char** argv)
|
|||||||
case 'e':
|
case 'e':
|
||||||
/* compression Level */
|
/* compression Level */
|
||||||
argument++;
|
argument++;
|
||||||
if ((*argument>='0') && (*argument<='9')) {
|
cLevelLast = readU32FromChar(&argument);
|
||||||
cLevelLast = 0;
|
|
||||||
while ((*argument >= '0') && (*argument <= '9'))
|
|
||||||
cLevelLast *= 10, cLevelLast += *argument++ - '0';
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
/* Modify Nb Iterations (benchmark only) */
|
/* Modify Nb Iterations (benchmark only) */
|
||||||
case 'i':
|
case 'i':
|
||||||
{ U32 iters= 0;
|
|
||||||
argument++;
|
argument++;
|
||||||
while ((*argument >='0') && (*argument <='9'))
|
{ U32 const iters = readU32FromChar(&argument);
|
||||||
iters *= 10, iters += *argument++ - '0';
|
|
||||||
BMK_setNotificationLevel(displayLevel);
|
BMK_setNotificationLevel(displayLevel);
|
||||||
BMK_SetNbIterations(iters);
|
BMK_SetNbIterations(iters);
|
||||||
}
|
}
|
||||||
@ -332,10 +348,8 @@ int main(int argCount, const char** argv)
|
|||||||
|
|
||||||
/* cut input into blocks (benchmark only) */
|
/* cut input into blocks (benchmark only) */
|
||||||
case 'B':
|
case 'B':
|
||||||
{ size_t bSize = 0;
|
|
||||||
argument++;
|
argument++;
|
||||||
while ((*argument >='0') && (*argument <='9'))
|
{ size_t bSize = readU32FromChar(&argument);
|
||||||
bSize *= 10, bSize += *argument++ - '0';
|
|
||||||
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
|
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
|
||||||
if (toupper(*argument)=='M') bSize<<=20, argument++;
|
if (toupper(*argument)=='M') bSize<<=20, argument++;
|
||||||
if (toupper(*argument)=='B') argument++;
|
if (toupper(*argument)=='B') argument++;
|
||||||
@ -345,21 +359,17 @@ int main(int argCount, const char** argv)
|
|||||||
break;
|
break;
|
||||||
#endif /* ZSTD_NOBENCH */
|
#endif /* ZSTD_NOBENCH */
|
||||||
|
|
||||||
/* Selection level */
|
/* Dictionary Selection level */
|
||||||
case 's': argument++;
|
case 's':
|
||||||
dictSelect = 0;
|
argument++;
|
||||||
while ((*argument >= '0') && (*argument <= '9'))
|
dictSelect = readU32FromChar(&argument);
|
||||||
dictSelect *= 10, dictSelect += *argument++ - '0';
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
|
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
|
||||||
case 'p': argument++;
|
case 'p': argument++;
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
if ((*argument>='0') && (*argument<='9')) {
|
if ((*argument>='0') && (*argument<='9')) {
|
||||||
int additionalParam = 0;
|
BMK_setAdditionalParam(readU32FromChar(&argument));
|
||||||
while ((*argument >= '0') && (*argument <= '9'))
|
|
||||||
additionalParam *= 10, additionalParam += *argument++ - '0';
|
|
||||||
BMK_setAdditionalParam(additionalParam);
|
|
||||||
} else
|
} else
|
||||||
#endif
|
#endif
|
||||||
main_pause=1;
|
main_pause=1;
|
||||||
@ -386,10 +396,15 @@ int main(int argCount, const char** argv)
|
|||||||
|
|
||||||
if (nextArgumentIsMaxDict) {
|
if (nextArgumentIsMaxDict) {
|
||||||
nextArgumentIsMaxDict = 0;
|
nextArgumentIsMaxDict = 0;
|
||||||
maxDictSize = 0;
|
maxDictSize = readU32FromChar(&argument);
|
||||||
while ((*argument>='0') && (*argument<='9'))
|
|
||||||
maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++;
|
|
||||||
if (toupper(*argument)=='K') maxDictSize <<= 10;
|
if (toupper(*argument)=='K') maxDictSize <<= 10;
|
||||||
|
if (toupper(*argument)=='M') maxDictSize <<= 20;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nextArgumentIsDictID) {
|
||||||
|
nextArgumentIsDictID = 0;
|
||||||
|
dictID = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -429,6 +444,7 @@ int main(int argCount, const char** argv)
|
|||||||
dictParams.compressionLevel = dictCLevel;
|
dictParams.compressionLevel = dictCLevel;
|
||||||
dictParams.selectivityLevel = dictSelect;
|
dictParams.selectivityLevel = dictSelect;
|
||||||
dictParams.notificationLevel = displayLevel;
|
dictParams.notificationLevel = displayLevel;
|
||||||
|
dictParams.dictID = dictID;
|
||||||
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
|
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
|
||||||
#endif
|
#endif
|
||||||
goto _end;
|
goto _end;
|
||||||
|
2
projects/.gitignore
vendored
Normal file
2
projects/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
build
|
||||||
|
*Copy
|
@ -92,80 +92,6 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
<Configuration
|
|
||||||
Name="Debug|x64"
|
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
|
||||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
|
||||||
ConfigurationType="1"
|
|
||||||
CharacterSet="2"
|
|
||||||
>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreBuildEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCustomBuildTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXMLDataGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCWebServiceProxyGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCMIDLTool"
|
|
||||||
TargetEnvironment="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCLCompilerTool"
|
|
||||||
Optimization="0"
|
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
|
||||||
MinimalRebuild="true"
|
|
||||||
BasicRuntimeChecks="3"
|
|
||||||
RuntimeLibrary="3"
|
|
||||||
UsePrecompiledHeader="0"
|
|
||||||
WarningLevel="4"
|
|
||||||
WarnAsError="true"
|
|
||||||
DebugInformationFormat="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManagedResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreLinkEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCLinkerTool"
|
|
||||||
LinkIncremental="2"
|
|
||||||
GenerateDebugInformation="true"
|
|
||||||
SubSystem="1"
|
|
||||||
TargetMachine="17"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCALinkTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManifestTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXDCMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCBscMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCFxCopTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCAppVerifierTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPostBuildEventTool"
|
|
||||||
/>
|
|
||||||
</Configuration>
|
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|Win32"
|
Name="Release|Win32"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -242,6 +168,80 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
|
<Configuration
|
||||||
|
Name="Debug|x64"
|
||||||
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
|
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||||
|
ConfigurationType="1"
|
||||||
|
CharacterSet="2"
|
||||||
|
>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreBuildEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCustomBuildTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXMLDataGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCWebServiceProxyGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCMIDLTool"
|
||||||
|
TargetEnvironment="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCLCompilerTool"
|
||||||
|
Optimization="0"
|
||||||
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
|
MinimalRebuild="true"
|
||||||
|
BasicRuntimeChecks="3"
|
||||||
|
RuntimeLibrary="3"
|
||||||
|
UsePrecompiledHeader="0"
|
||||||
|
WarningLevel="4"
|
||||||
|
WarnAsError="true"
|
||||||
|
DebugInformationFormat="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManagedResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreLinkEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCLinkerTool"
|
||||||
|
LinkIncremental="2"
|
||||||
|
GenerateDebugInformation="true"
|
||||||
|
SubSystem="1"
|
||||||
|
TargetMachine="17"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCALinkTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManifestTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXDCMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCBscMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCFxCopTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCAppVerifierTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPostBuildEventTool"
|
||||||
|
/>
|
||||||
|
</Configuration>
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|x64"
|
Name="Release|x64"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -344,6 +344,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\programs\fullbench.c"
|
RelativePath="..\..\..\programs\fullbench.c"
|
||||||
>
|
>
|
||||||
@ -410,6 +414,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\huf_static.h"
|
RelativePath="..\..\..\lib\common\huf_static.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\mem.h"
|
RelativePath="..\..\..\lib\common\mem.h"
|
||||||
>
|
>
|
||||||
|
@ -44,7 +44,7 @@
|
|||||||
<Tool
|
<Tool
|
||||||
Name="VCCLCompilerTool"
|
Name="VCCLCompilerTool"
|
||||||
Optimization="0"
|
Optimization="0"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
MinimalRebuild="true"
|
MinimalRebuild="true"
|
||||||
BasicRuntimeChecks="3"
|
BasicRuntimeChecks="3"
|
||||||
@ -92,80 +92,6 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
<Configuration
|
|
||||||
Name="Debug|x64"
|
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
|
||||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
|
||||||
ConfigurationType="1"
|
|
||||||
CharacterSet="2"
|
|
||||||
>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreBuildEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCustomBuildTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXMLDataGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCWebServiceProxyGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCMIDLTool"
|
|
||||||
TargetEnvironment="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCLCompilerTool"
|
|
||||||
Optimization="0"
|
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
|
||||||
MinimalRebuild="true"
|
|
||||||
BasicRuntimeChecks="3"
|
|
||||||
RuntimeLibrary="3"
|
|
||||||
UsePrecompiledHeader="0"
|
|
||||||
WarningLevel="4"
|
|
||||||
WarnAsError="true"
|
|
||||||
DebugInformationFormat="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManagedResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreLinkEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCLinkerTool"
|
|
||||||
LinkIncremental="2"
|
|
||||||
GenerateDebugInformation="true"
|
|
||||||
SubSystem="1"
|
|
||||||
TargetMachine="17"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCALinkTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManifestTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXDCMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCBscMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCFxCopTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCAppVerifierTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPostBuildEventTool"
|
|
||||||
/>
|
|
||||||
</Configuration>
|
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|Win32"
|
Name="Release|Win32"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -194,7 +120,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -242,6 +168,80 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
|
<Configuration
|
||||||
|
Name="Debug|x64"
|
||||||
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
|
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||||
|
ConfigurationType="1"
|
||||||
|
CharacterSet="2"
|
||||||
|
>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreBuildEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCustomBuildTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXMLDataGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCWebServiceProxyGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCMIDLTool"
|
||||||
|
TargetEnvironment="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCLCompilerTool"
|
||||||
|
Optimization="0"
|
||||||
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
|
MinimalRebuild="true"
|
||||||
|
BasicRuntimeChecks="3"
|
||||||
|
RuntimeLibrary="3"
|
||||||
|
UsePrecompiledHeader="0"
|
||||||
|
WarningLevel="4"
|
||||||
|
WarnAsError="true"
|
||||||
|
DebugInformationFormat="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManagedResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreLinkEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCLinkerTool"
|
||||||
|
LinkIncremental="2"
|
||||||
|
GenerateDebugInformation="true"
|
||||||
|
SubSystem="1"
|
||||||
|
TargetMachine="17"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCALinkTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManifestTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXDCMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCBscMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCFxCopTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCAppVerifierTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPostBuildEventTool"
|
||||||
|
/>
|
||||||
|
</Configuration>
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|x64"
|
Name="Release|x64"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -271,7 +271,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -332,6 +332,10 @@
|
|||||||
RelativePath="..\..\..\programs\datagen.c"
|
RelativePath="..\..\..\programs\datagen.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||||
>
|
>
|
||||||
@ -357,7 +361,11 @@
|
|||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\programs\xxhash.c"
|
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
<File
|
<File
|
||||||
@ -382,6 +390,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\error_private.h"
|
RelativePath="..\..\..\lib\common\error_private.h"
|
||||||
>
|
>
|
||||||
@ -410,10 +422,22 @@
|
|||||||
RelativePath="..\..\..\lib\common\mem.h"
|
RelativePath="..\..\..\lib\common\mem.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\zstd.h"
|
RelativePath="..\..\..\lib\common\zstd.h"
|
||||||
>
|
>
|
||||||
|
@ -44,7 +44,7 @@
|
|||||||
<Tool
|
<Tool
|
||||||
Name="VCCLCompilerTool"
|
Name="VCCLCompilerTool"
|
||||||
Optimization="0"
|
Optimization="0"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
MinimalRebuild="true"
|
MinimalRebuild="true"
|
||||||
BasicRuntimeChecks="3"
|
BasicRuntimeChecks="3"
|
||||||
@ -93,81 +93,6 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
<Configuration
|
|
||||||
Name="Debug|x64"
|
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
|
||||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
|
||||||
ConfigurationType="1"
|
|
||||||
CharacterSet="2"
|
|
||||||
>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreBuildEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCustomBuildTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXMLDataGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCWebServiceProxyGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCMIDLTool"
|
|
||||||
TargetEnvironment="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCLCompilerTool"
|
|
||||||
Optimization="0"
|
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
|
||||||
MinimalRebuild="true"
|
|
||||||
BasicRuntimeChecks="3"
|
|
||||||
RuntimeLibrary="3"
|
|
||||||
UsePrecompiledHeader="0"
|
|
||||||
WarningLevel="4"
|
|
||||||
WarnAsError="true"
|
|
||||||
DebugInformationFormat="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManagedResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreLinkEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCLinkerTool"
|
|
||||||
AdditionalDependencies="setargv.obj"
|
|
||||||
LinkIncremental="2"
|
|
||||||
GenerateDebugInformation="true"
|
|
||||||
SubSystem="1"
|
|
||||||
TargetMachine="17"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCALinkTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManifestTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXDCMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCBscMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCFxCopTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCAppVerifierTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPostBuildEventTool"
|
|
||||||
/>
|
|
||||||
</Configuration>
|
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|Win32"
|
Name="Release|Win32"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -196,7 +121,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -245,6 +170,81 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
|
<Configuration
|
||||||
|
Name="Debug|x64"
|
||||||
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
|
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||||
|
ConfigurationType="1"
|
||||||
|
CharacterSet="2"
|
||||||
|
>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreBuildEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCustomBuildTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXMLDataGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCWebServiceProxyGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCMIDLTool"
|
||||||
|
TargetEnvironment="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCLCompilerTool"
|
||||||
|
Optimization="0"
|
||||||
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
|
MinimalRebuild="true"
|
||||||
|
BasicRuntimeChecks="3"
|
||||||
|
RuntimeLibrary="3"
|
||||||
|
UsePrecompiledHeader="0"
|
||||||
|
WarningLevel="4"
|
||||||
|
WarnAsError="true"
|
||||||
|
DebugInformationFormat="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManagedResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreLinkEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCLinkerTool"
|
||||||
|
AdditionalDependencies="setargv.obj"
|
||||||
|
LinkIncremental="2"
|
||||||
|
GenerateDebugInformation="true"
|
||||||
|
SubSystem="1"
|
||||||
|
TargetMachine="17"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCALinkTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManifestTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXDCMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCBscMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCFxCopTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCAppVerifierTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPostBuildEventTool"
|
||||||
|
/>
|
||||||
|
</Configuration>
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|x64"
|
Name="Release|x64"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -274,7 +274,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -377,7 +377,7 @@
|
|||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\programs\xxhash.c"
|
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
<File
|
<File
|
||||||
@ -470,6 +470,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\mem.h"
|
RelativePath="..\..\..\lib\common\mem.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||||
>
|
>
|
||||||
|
@ -44,7 +44,7 @@
|
|||||||
<Tool
|
<Tool
|
||||||
Name="VCCLCompilerTool"
|
Name="VCCLCompilerTool"
|
||||||
Optimization="0"
|
Optimization="0"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
||||||
MinimalRebuild="true"
|
MinimalRebuild="true"
|
||||||
BasicRuntimeChecks="3"
|
BasicRuntimeChecks="3"
|
||||||
@ -92,80 +92,6 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
<Configuration
|
|
||||||
Name="Debug|x64"
|
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
|
||||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
|
||||||
ConfigurationType="2"
|
|
||||||
CharacterSet="2"
|
|
||||||
>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreBuildEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCustomBuildTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXMLDataGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCWebServiceProxyGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCMIDLTool"
|
|
||||||
TargetEnvironment="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCLCompilerTool"
|
|
||||||
Optimization="0"
|
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
|
||||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
|
||||||
MinimalRebuild="true"
|
|
||||||
BasicRuntimeChecks="3"
|
|
||||||
RuntimeLibrary="3"
|
|
||||||
UsePrecompiledHeader="0"
|
|
||||||
WarningLevel="4"
|
|
||||||
WarnAsError="true"
|
|
||||||
DebugInformationFormat="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManagedResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreLinkEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCLinkerTool"
|
|
||||||
LinkIncremental="2"
|
|
||||||
GenerateDebugInformation="true"
|
|
||||||
SubSystem="1"
|
|
||||||
TargetMachine="17"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCALinkTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManifestTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXDCMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCBscMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCFxCopTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCAppVerifierTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPostBuildEventTool"
|
|
||||||
/>
|
|
||||||
</Configuration>
|
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|Win32"
|
Name="Release|Win32"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -194,7 +120,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -242,6 +168,80 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
|
<Configuration
|
||||||
|
Name="Debug|x64"
|
||||||
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
|
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||||
|
ConfigurationType="2"
|
||||||
|
CharacterSet="2"
|
||||||
|
>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreBuildEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCustomBuildTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXMLDataGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCWebServiceProxyGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCMIDLTool"
|
||||||
|
TargetEnvironment="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCLCompilerTool"
|
||||||
|
Optimization="0"
|
||||||
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
|
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
||||||
|
MinimalRebuild="true"
|
||||||
|
BasicRuntimeChecks="3"
|
||||||
|
RuntimeLibrary="3"
|
||||||
|
UsePrecompiledHeader="0"
|
||||||
|
WarningLevel="4"
|
||||||
|
WarnAsError="true"
|
||||||
|
DebugInformationFormat="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManagedResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreLinkEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCLinkerTool"
|
||||||
|
LinkIncremental="2"
|
||||||
|
GenerateDebugInformation="true"
|
||||||
|
SubSystem="1"
|
||||||
|
TargetMachine="17"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCALinkTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManifestTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXDCMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCBscMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCFxCopTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCAppVerifierTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPostBuildEventTool"
|
||||||
|
/>
|
||||||
|
</Configuration>
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|x64"
|
Name="Release|x64"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@ -271,7 +271,7 @@
|
|||||||
Optimization="2"
|
Optimization="2"
|
||||||
EnableIntrinsicFunctions="true"
|
EnableIntrinsicFunctions="true"
|
||||||
OmitFramePointers="true"
|
OmitFramePointers="true"
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
||||||
RuntimeLibrary="0"
|
RuntimeLibrary="0"
|
||||||
EnableFunctionLevelLinking="true"
|
EnableFunctionLevelLinking="true"
|
||||||
@ -352,6 +352,10 @@
|
|||||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||||
>
|
>
|
||||||
@ -418,6 +422,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\mem.h"
|
RelativePath="..\..\..\lib\common\mem.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||||
>
|
>
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|Win32">
|
<ProjectConfiguration Include="Debug|Win32">
|
||||||
@ -158,6 +158,7 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||||
@ -174,6 +175,7 @@
|
|||||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|Win32">
|
<ProjectConfiguration Include="Debug|Win32">
|
||||||
@ -66,24 +66,24 @@
|
|||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||||
<ClCompile>
|
<ClCompile>
|
||||||
@ -158,30 +158,35 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||||
<ClCompile Include="..\..\..\programs\fuzzer.c" />
|
<ClCompile Include="..\..\..\programs\fuzzer.c" />
|
||||||
<ClCompile Include="..\..\..\programs\xxhash.c" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||||
<ClInclude Include="..\..\..\programs\xxhash.h" />
|
|
||||||
<ClInclude Include="..\..\..\programs\util.h" />
|
<ClInclude Include="..\..\..\programs\util.h" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|Win32">
|
<ProjectConfiguration Include="Debug|Win32">
|
||||||
@ -20,6 +20,7 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||||
@ -41,10 +42,10 @@
|
|||||||
<ClCompile Include="..\..\..\programs\dibio.c" />
|
<ClCompile Include="..\..\..\programs\dibio.c" />
|
||||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
|
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
|
||||||
<ClCompile Include="..\..\..\programs\xxhash.c" />
|
|
||||||
<ClCompile Include="..\..\..\programs\zstdcli.c" />
|
<ClCompile Include="..\..\..\programs\zstdcli.c" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||||
@ -69,7 +70,6 @@
|
|||||||
<ClInclude Include="..\..\..\programs\dibio.h" />
|
<ClInclude Include="..\..\..\programs\dibio.h" />
|
||||||
<ClInclude Include="..\..\..\programs\fileio.h" />
|
<ClInclude Include="..\..\..\programs\fileio.h" />
|
||||||
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
|
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
|
||||||
<ClInclude Include="..\..\..\programs\xxhash.h" />
|
|
||||||
<ClInclude Include="..\..\..\programs\util.h" />
|
<ClInclude Include="..\..\..\programs\util.h" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<PropertyGroup Label="Globals">
|
<PropertyGroup Label="Globals">
|
||||||
@ -119,27 +119,27 @@
|
|||||||
<PropertyGroup Label="UserMacros" />
|
<PropertyGroup Label="UserMacros" />
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<LibraryPath>$(LibraryPath)</LibraryPath>
|
<LibraryPath>$(LibraryPath)</LibraryPath>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<LibraryPath>$(LibraryPath);</LibraryPath>
|
<LibraryPath>$(LibraryPath);</LibraryPath>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<LibraryPath>$(LibraryPath)</LibraryPath>
|
<LibraryPath>$(LibraryPath)</LibraryPath>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
<LibraryPath>$(LibraryPath);</LibraryPath>
|
<LibraryPath>$(LibraryPath);</LibraryPath>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
51
projects/VS2010/zstdlib/zstdlib.rc
Normal file
51
projects/VS2010/zstdlib/zstdlib.rc
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
// Microsoft Visual C++ generated resource script.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||||
|
#define APSTUDIO_READONLY_SYMBOLS
|
||||||
|
#include "verrsrc.h"
|
||||||
|
#undef APSTUDIO_READONLY_SYMBOLS
|
||||||
|
|
||||||
|
|
||||||
|
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
|
||||||
|
LANGUAGE 9, 1
|
||||||
|
|
||||||
|
/////////////////////////////////////////////////////////////////////////////
|
||||||
|
//
|
||||||
|
// Version
|
||||||
|
//
|
||||||
|
|
||||||
|
VS_VERSION_INFO VERSIONINFO
|
||||||
|
FILEVERSION ZSTD_LIB_VERSION
|
||||||
|
PRODUCTVERSION ZSTD_LIB_VERSION
|
||||||
|
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
|
||||||
|
#ifdef _DEBUG
|
||||||
|
FILEFLAGS VS_FF_DEBUG
|
||||||
|
#else
|
||||||
|
FILEFLAGS 0x0L
|
||||||
|
#endif
|
||||||
|
FILEOS VOS_NT_WINDOWS32
|
||||||
|
FILETYPE VFT_DLL
|
||||||
|
FILESUBTYPE VFT2_UNKNOWN
|
||||||
|
BEGIN
|
||||||
|
BLOCK "StringFileInfo"
|
||||||
|
BEGIN
|
||||||
|
BLOCK "040904B0"
|
||||||
|
BEGIN
|
||||||
|
VALUE "CompanyName", "Yann Collet"
|
||||||
|
VALUE "FileDescription", "Fast and efficient compression algorithm"
|
||||||
|
VALUE "FileVersion", ZSTD_VERSION_STRING
|
||||||
|
VALUE "InternalName", "zstdlib.dll"
|
||||||
|
VALUE "LegalCopyright", "Copyright (C) 2013-2015, Yann Collet"
|
||||||
|
VALUE "OriginalFilename", "zstdlib.dll"
|
||||||
|
VALUE "ProductName", "Zstandard"
|
||||||
|
VALUE "ProductVersion", ZSTD_VERSION_STRING
|
||||||
|
END
|
||||||
|
END
|
||||||
|
BLOCK "VarFileInfo"
|
||||||
|
BEGIN
|
||||||
|
VALUE "Translation", 0x0409, 1200
|
||||||
|
END
|
||||||
|
END
|
||||||
|
|
||||||
|
#endif
|
@ -1,4 +1,4 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|Win32">
|
<ProjectConfiguration Include="Debug|Win32">
|
||||||
@ -20,6 +20,7 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||||
@ -39,6 +40,7 @@
|
|||||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||||
@ -46,7 +48,9 @@
|
|||||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||||
<ClInclude Include="..\..\..\programs\util.h" />
|
<ClInclude Include="..\..\..\programs\util.h" />
|
||||||
<ClInclude Include="resource.h" />
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<ResourceCompile Include="zstdlib.rc" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<PropertyGroup Label="Globals">
|
<PropertyGroup Label="Globals">
|
||||||
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
|
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
|
||||||
@ -97,28 +101,28 @@
|
|||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<TargetName>zstdlib_x86</TargetName>
|
<TargetName>zstdlib_x86</TargetName>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||||
<LinkIncremental>true</LinkIncremental>
|
<LinkIncremental>true</LinkIncremental>
|
||||||
<TargetName>zstdlib_x64</TargetName>
|
<TargetName>zstdlib_x64</TargetName>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<TargetName>zstdlib_x86</TargetName>
|
<TargetName>zstdlib_x86</TargetName>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||||
<LinkIncremental>false</LinkIncremental>
|
<LinkIncremental>false</LinkIncremental>
|
||||||
<TargetName>zstdlib_x64</TargetName>
|
<TargetName>zstdlib_x64</TargetName>
|
||||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||||
|
6
projects/cmake/.gitignore
vendored
Normal file
6
projects/cmake/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
# cmake producted
|
||||||
|
CMakeCache.txt
|
||||||
|
CMakeFiles
|
||||||
|
Makefile
|
||||||
|
cmake_install.cmake
|
||||||
|
cmake_uninstall.cmake
|
@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT)
|
|||||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
||||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||||
|
|
||||||
# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate
|
|
||||||
IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
|
|
||||||
SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE)
|
|
||||||
ELSE ()
|
|
||||||
SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE)
|
|
||||||
ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
|
|
||||||
|
|
||||||
ADD_SUBDIRECTORY(lib)
|
ADD_SUBDIRECTORY(lib)
|
||||||
ADD_SUBDIRECTORY(programs)
|
ADD_SUBDIRECTORY(programs)
|
||||||
|
|
||||||
@ -59,3 +52,8 @@ ADD_SUBDIRECTORY(programs)
|
|||||||
#-----------------------------------------------------------------------------
|
#-----------------------------------------------------------------------------
|
||||||
INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
|
INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
|
||||||
ADD_EXTRA_COMPILATION_FLAGS()
|
ADD_EXTRA_COMPILATION_FLAGS()
|
||||||
|
|
||||||
|
ADD_CUSTOM_TARGET(clean-all
|
||||||
|
COMMAND ${CMAKE_BUILD_TOOL} clean
|
||||||
|
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
|
||||||
|
)
|
||||||
|
@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
|||||||
SET(Sources
|
SET(Sources
|
||||||
${LIBRARY_DIR}/common/entropy_common.c
|
${LIBRARY_DIR}/common/entropy_common.c
|
||||||
${LIBRARY_DIR}/common/zstd_common.c
|
${LIBRARY_DIR}/common/zstd_common.c
|
||||||
|
${LIBRARY_DIR}/common/xxhash.c
|
||||||
${LIBRARY_DIR}/common/fse_decompress.c
|
${LIBRARY_DIR}/common/fse_decompress.c
|
||||||
${LIBRARY_DIR}/compress/fse_compress.c
|
${LIBRARY_DIR}/compress/fse_compress.c
|
||||||
${LIBRARY_DIR}/compress/huf_compress.c
|
${LIBRARY_DIR}/compress/huf_compress.c
|
||||||
@ -75,17 +76,12 @@ SET(Headers
|
|||||||
${LIBRARY_DIR}/common/error_private.h
|
${LIBRARY_DIR}/common/error_private.h
|
||||||
${LIBRARY_DIR}/common/error_public.h
|
${LIBRARY_DIR}/common/error_public.h
|
||||||
${LIBRARY_DIR}/common/fse.h
|
${LIBRARY_DIR}/common/fse.h
|
||||||
${LIBRARY_DIR}/common/fse_static.h
|
|
||||||
${LIBRARY_DIR}/common/huf.h
|
${LIBRARY_DIR}/common/huf.h
|
||||||
${LIBRARY_DIR}/common/huf_static.h
|
|
||||||
${LIBRARY_DIR}/common/mem.h
|
${LIBRARY_DIR}/common/mem.h
|
||||||
${LIBRARY_DIR}/common/zbuff.h
|
${LIBRARY_DIR}/common/zbuff.h
|
||||||
${LIBRARY_DIR}/common/zbuff_static.h
|
|
||||||
${LIBRARY_DIR}/common/zstd_internal.h
|
${LIBRARY_DIR}/common/zstd_internal.h
|
||||||
${LIBRARY_DIR}/common/zstd_static.h
|
|
||||||
${LIBRARY_DIR}/common/zstd.h
|
${LIBRARY_DIR}/common/zstd.h
|
||||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
${LIBRARY_DIR}/dictBuilder/zdict.h)
|
||||||
${LIBRARY_DIR}/dictBuilder/zdict_static.h)
|
|
||||||
|
|
||||||
IF (ZSTD_LEGACY_SUPPORT)
|
IF (ZSTD_LEGACY_SUPPORT)
|
||||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||||
@ -95,19 +91,21 @@ IF (ZSTD_LEGACY_SUPPORT)
|
|||||||
${LIBRARY_LEGACY_DIR}/zstd_v01.c
|
${LIBRARY_LEGACY_DIR}/zstd_v01.c
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v02.c
|
${LIBRARY_LEGACY_DIR}/zstd_v02.c
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v04.c)
|
${LIBRARY_LEGACY_DIR}/zstd_v04.c
|
||||||
|
${LIBRARY_LEGACY_DIR}/zstd_v05.c)
|
||||||
|
|
||||||
SET(Headers ${Headers}
|
SET(Headers ${Headers}
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v01.h
|
${LIBRARY_LEGACY_DIR}/zstd_v01.h
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v02.h
|
${LIBRARY_LEGACY_DIR}/zstd_v02.h
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v03.h
|
${LIBRARY_LEGACY_DIR}/zstd_v03.h
|
||||||
${LIBRARY_LEGACY_DIR}/zstd_v04.h)
|
${LIBRARY_LEGACY_DIR}/zstd_v04.h
|
||||||
|
${LIBRARY_LEGACY_DIR}/zstd_v05.h)
|
||||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||||
|
|
||||||
IF (MSVC)
|
IF (MSVC)
|
||||||
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib)
|
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib)
|
||||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc)
|
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
|
||||||
ENDIF (MSVC)
|
ENDIF (MSVC)
|
||||||
|
|
||||||
# Split project to static and shared libraries build
|
# Split project to static and shared libraries build
|
||||||
@ -120,24 +118,12 @@ IF (MSVC)
|
|||||||
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||||
ENDIF (MSVC)
|
ENDIF (MSVC)
|
||||||
|
|
||||||
# Define include directories
|
|
||||||
IF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common)
|
|
||||||
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common)
|
|
||||||
IF (ZSTD_LEGACY_SUPPORT)
|
|
||||||
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR})
|
|
||||||
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR})
|
|
||||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
|
||||||
ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
|
|
||||||
# Define library base name
|
# Define library base name
|
||||||
IF (UNIX)
|
IF (MSVC)
|
||||||
SET(LIBRARY_BASE_NAME libzstd)
|
|
||||||
ELSEIF (MSVC)
|
|
||||||
SET(LIBRARY_BASE_NAME zstdlib)
|
SET(LIBRARY_BASE_NAME zstdlib)
|
||||||
ELSE ()
|
ELSE ()
|
||||||
MESSAGE(FATAL_ERROR "Unsupported build type")
|
SET(LIBRARY_BASE_NAME libzstd)
|
||||||
ENDIF (UNIX)
|
ENDIF (MSVC)
|
||||||
|
|
||||||
# Define static and shared library names
|
# Define static and shared library names
|
||||||
SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME})
|
SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME})
|
||||||
@ -166,12 +152,15 @@ SET_TARGET_PROPERTIES(
|
|||||||
OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME})
|
OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME})
|
||||||
|
|
||||||
IF (UNIX)
|
IF (UNIX)
|
||||||
|
IF ("${PREFIX}" STREQUAL "")
|
||||||
SET(PREFIX /usr/local)
|
SET(PREFIX /usr/local)
|
||||||
|
ENDIF()
|
||||||
|
MESSAGE("the variable PREFIX=${PREFIX}")
|
||||||
SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib)
|
SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib)
|
||||||
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
|
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
|
||||||
|
|
||||||
# install target
|
# install target
|
||||||
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/zstd_buffered.h ${LIBRARY_DIR}/dictBuilder.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||||
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
|
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||||
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
|
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||||
|
|
||||||
|
8
projects/cmake/programs/.gitignore
vendored
Normal file
8
projects/cmake/programs/.gitignore
vendored
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
# produced by make
|
||||||
|
datagen
|
||||||
|
fullbench
|
||||||
|
fuzzer
|
||||||
|
paramgrill
|
||||||
|
zbufftest
|
||||||
|
zstd
|
||||||
|
zstd-frugal
|
@ -38,33 +38,23 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
|||||||
SET(ROOT_DIR ../../..)
|
SET(ROOT_DIR ../../..)
|
||||||
|
|
||||||
# Define programs directory, where sources and header files are located
|
# Define programs directory, where sources and header files are located
|
||||||
|
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||||
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
|
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
|
||||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR})
|
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
|
||||||
|
|
||||||
IF (WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
# Define library directory, where sources and header files are located
|
|
||||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
|
||||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common)
|
|
||||||
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
|
|
||||||
IF (ZSTD_LEGACY_SUPPORT)
|
IF (ZSTD_LEGACY_SUPPORT)
|
||||||
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR})
|
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||||
|
|
||||||
IF (WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy)
|
|
||||||
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
|
|
||||||
|
|
||||||
SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
|
SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
|
||||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||||
|
|
||||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
|
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
|
||||||
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
||||||
|
|
||||||
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
|
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
|
||||||
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
||||||
|
|
||||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/fuzzer.c)
|
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c)
|
||||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||||
|
|
||||||
IF (UNIX)
|
IF (UNIX)
|
||||||
@ -72,10 +62,10 @@ IF (UNIX)
|
|||||||
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
||||||
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||||
|
|
||||||
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/zbufftest.c)
|
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c)
|
||||||
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
|
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
|
||||||
|
|
||||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/paramgrill.c)
|
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c)
|
||||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||||
|
|
||||||
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)
|
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)
|
||||||
|
@ -62,6 +62,7 @@ def compress_sample(tag, sample):
|
|||||||
os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
|
os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
|
||||||
# zstdFiles = glob.glob("*.zst*")
|
# zstdFiles = glob.glob("*.zst*")
|
||||||
# print(zstdFiles)
|
# print(zstdFiles)
|
||||||
|
print(tag + " : compression completed")
|
||||||
|
|
||||||
|
|
||||||
# http://stackoverflow.com/a/19711609/2132223
|
# http://stackoverflow.com/a/19711609/2132223
|
||||||
|
3
zlibWrapper/.gitignore
vendored
3
zlibWrapper/.gitignore
vendored
@ -19,6 +19,9 @@
|
|||||||
|
|
||||||
# Default result files
|
# Default result files
|
||||||
_*
|
_*
|
||||||
|
example
|
||||||
|
example_zstd
|
||||||
|
foo.gz
|
||||||
|
|
||||||
# Misc files
|
# Misc files
|
||||||
*.bat
|
*.bat
|
||||||
|
@ -6,23 +6,25 @@
|
|||||||
|
|
||||||
|
|
||||||
# Paths to static and dynamic zlib and zstd libraries
|
# Paths to static and dynamic zlib and zstd libraries
|
||||||
ifneq (,$(filter Windows%,$(OS)))
|
# Use "make ZLIBDIR=path/to/zlib" to select a path to library
|
||||||
STATICLIB = ../../zlib/libz.a ../lib/libzstd.a
|
ifdef ZLIBDIR
|
||||||
IMPLIB = ../../zlib/libz.dll.a ../lib/libzstd.a
|
STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a
|
||||||
|
IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a
|
||||||
else
|
else
|
||||||
STATICLIB = -static -lz ../lib/libzstd.a
|
STATICLIB = -static -lz ../lib/libzstd.a
|
||||||
IMPLIB = -lz ../lib/libzstd.a
|
IMPLIB = -lz ../lib/libzstd.a
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ZLIBWRAPPER_PATH = .
|
ZLIBWRAPPER_PATH = .
|
||||||
EXAMPLE_PATH = examples/
|
EXAMPLE_PATH = examples
|
||||||
CC = gcc
|
CC = gcc
|
||||||
CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89
|
CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90
|
||||||
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||||
LDFLAGS = $(LOC)
|
LDFLAGS = $(LOC)
|
||||||
RM = rm -f
|
RM = rm -f
|
||||||
|
|
||||||
|
|
||||||
all: example example_d
|
all: clean test testzstd
|
||||||
|
|
||||||
test: example
|
test: example
|
||||||
./example
|
./example
|
||||||
@ -30,6 +32,9 @@ test: example
|
|||||||
testdll: example_d
|
testdll: example_d
|
||||||
./example_d
|
./example_d
|
||||||
|
|
||||||
|
testzstd: example_zstd
|
||||||
|
./example_zstd
|
||||||
|
|
||||||
.c.o:
|
.c.o:
|
||||||
$(CC) $(CFLAGS) -c -o $@ $<
|
$(CC) $(CFLAGS) -c -o $@ $<
|
||||||
|
|
||||||
@ -39,16 +44,18 @@ example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
|
|||||||
example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
|
example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
|
||||||
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)
|
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)
|
||||||
|
|
||||||
|
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o
|
||||||
|
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB)
|
||||||
|
|
||||||
$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
|
$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
|
||||||
$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c
|
$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c
|
||||||
|
|
||||||
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||||
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||||
|
|
||||||
clean:
|
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||||
-$(RM) $(ZLIBWRAPPER_PATH)/*.o
|
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||||
-$(RM) $(EXAMPLE_PATH)/*.o
|
|
||||||
-$(RM) *.o
|
|
||||||
-$(RM) *.exe
|
|
||||||
-$(RM) foo.gz
|
|
||||||
|
|
||||||
|
clean:
|
||||||
|
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd
|
||||||
|
@echo Cleaning completed
|
||||||
|
@ -61,12 +61,15 @@ void *myalloc(q, n, m)
|
|||||||
void *q;
|
void *q;
|
||||||
unsigned n, m;
|
unsigned n, m;
|
||||||
{
|
{
|
||||||
|
void *buf = calloc(n, m);
|
||||||
q = Z_NULL;
|
q = Z_NULL;
|
||||||
return calloc(n, m);
|
/* printf("myalloc %p n=%d m=%d\n", buf, n, m); */
|
||||||
|
return buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
void myfree(void *q, void *p)
|
void myfree(void *q, void *p)
|
||||||
{
|
{
|
||||||
|
/* printf("myfree %p\n", p); */
|
||||||
q = Z_NULL;
|
q = Z_NULL;
|
||||||
free(p);
|
free(p);
|
||||||
}
|
}
|
||||||
|
@ -29,30 +29,35 @@
|
|||||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include <stdio.h> /* fprintf */
|
#include <stdarg.h> /* va_list, for z_gzprintf */
|
||||||
#include <stdlib.h> /* malloc */
|
|
||||||
#include <zlib.h>
|
#include <zlib.h>
|
||||||
#include "zstd_zlibwrapper.h"
|
#include "zstd_zlibwrapper.h"
|
||||||
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
|
||||||
#include "zstd.h"
|
#include "zstd.h"
|
||||||
#include "zstd_static.h" /* ZSTD_MAGICNUMBER */
|
#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
|
||||||
#include "zstd_internal.h" /* MIN */
|
|
||||||
#include "zbuff.h"
|
#include "zbuff.h"
|
||||||
|
#include "zstd_internal.h" /* defaultCustomMem */
|
||||||
|
|
||||||
|
|
||||||
#define Z_INFLATE_SYNC 8
|
#define Z_INFLATE_SYNC 8
|
||||||
#define ZWRAP_HEADERSIZE 4
|
#define ZWRAP_HEADERSIZE 4
|
||||||
#define ZSTD_FRAMEHEADERSIZE_MIN 5
|
#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
|
||||||
|
|
||||||
#define LOG_WRAPPER(...) // printf(__VA_ARGS__)
|
#define LOG_WRAPPER(...) // printf(__VA_ARGS__)
|
||||||
|
|
||||||
|
|
||||||
#define FINISH_WITH_ERR(msg) { \
|
#define FINISH_WITH_GZ_ERR(msg) { \
|
||||||
fprintf(stderr, "ERROR: %s\n", msg); \
|
(void)msg; \
|
||||||
|
return Z_MEM_ERROR; \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define FINISH_WITH_ERR(strm, message) { \
|
||||||
|
strm->msg = message; \
|
||||||
return Z_MEM_ERROR; \
|
return Z_MEM_ERROR; \
|
||||||
}
|
}
|
||||||
|
|
||||||
#define FINISH_WITH_NULL_ERR(msg) { \
|
#define FINISH_WITH_NULL_ERR(msg) { \
|
||||||
fprintf(stderr, "ERROR: %s\n", msg); \
|
(void)msg; \
|
||||||
return NULL; \
|
return NULL; \
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -66,13 +71,29 @@ static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
|
|||||||
|
|
||||||
void useZSTD(int turn_on) { g_useZSTD = turn_on; }
|
void useZSTD(int turn_on) { g_useZSTD = turn_on; }
|
||||||
|
|
||||||
int isUsingZSTD() { return g_useZSTD; }
|
int isUsingZSTD(void) { return g_useZSTD; }
|
||||||
|
|
||||||
const char * zstdVersion() { return ZSTD_VERSION_STRING; }
|
const char * zstdVersion(void) { return ZSTD_VERSION_STRING; }
|
||||||
|
|
||||||
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
|
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
|
||||||
|
|
||||||
|
|
||||||
|
static void* ZWRAP_allocFunction(void* opaque, size_t size)
|
||||||
|
{
|
||||||
|
z_streamp strm = (z_streamp) opaque;
|
||||||
|
void* address = strm->zalloc(strm->opaque, 1, size);
|
||||||
|
/* printf("ZWRAP alloc %p, %d \n", address, (int)size); */
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZWRAP_freeFunction(void* opaque, void* address)
|
||||||
|
{
|
||||||
|
z_streamp strm = (z_streamp) opaque;
|
||||||
|
strm->zfree(strm->opaque, address);
|
||||||
|
/* if (address) printf("ZWRAP free %p \n", address); */
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* *** Compression *** */
|
/* *** Compression *** */
|
||||||
|
|
||||||
@ -80,28 +101,45 @@ typedef struct {
|
|||||||
ZBUFF_CCtx* zbc;
|
ZBUFF_CCtx* zbc;
|
||||||
size_t bytesLeft;
|
size_t bytesLeft;
|
||||||
int compressionLevel;
|
int compressionLevel;
|
||||||
|
ZSTD_customMem customMem;
|
||||||
|
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
|
||||||
} ZWRAP_CCtx;
|
} ZWRAP_CCtx;
|
||||||
|
|
||||||
|
|
||||||
ZWRAP_CCtx* ZWRAP_createCCtx()
|
|
||||||
{
|
|
||||||
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx));
|
|
||||||
if (zwc==NULL) return NULL;
|
|
||||||
memset(zwc, 0, sizeof(*zwc));
|
|
||||||
zwc->zbc = ZBUFF_createCCtx();
|
|
||||||
return zwc;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
|
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
|
||||||
{
|
{
|
||||||
if (zwc==NULL) return 0; /* support free on NULL */
|
if (zwc==NULL) return 0; /* support free on NULL */
|
||||||
ZBUFF_freeCCtx(zwc->zbc);
|
ZBUFF_freeCCtx(zwc->zbc);
|
||||||
free(zwc);
|
zwc->customMem.customFree(zwc->customMem.opaque, zwc);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
|
||||||
|
{
|
||||||
|
ZWRAP_CCtx* zwc;
|
||||||
|
|
||||||
|
if (strm->zalloc && strm->zfree) {
|
||||||
|
zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx));
|
||||||
|
if (zwc==NULL) return NULL;
|
||||||
|
memset(zwc, 0, sizeof(ZWRAP_CCtx));
|
||||||
|
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
|
||||||
|
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
|
||||||
|
memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx));
|
||||||
|
if (zwc==NULL) return NULL;
|
||||||
|
memset(zwc, 0, sizeof(ZWRAP_CCtx));
|
||||||
|
memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||||
|
}
|
||||||
|
|
||||||
|
zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem);
|
||||||
|
if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; }
|
||||||
|
return zwc;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
|
ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
|
||||||
const char *version, int stream_size))
|
const char *version, int stream_size))
|
||||||
{
|
{
|
||||||
@ -113,14 +151,17 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
|
|||||||
}
|
}
|
||||||
|
|
||||||
LOG_WRAPPER("- deflateInit level=%d\n", level);
|
LOG_WRAPPER("- deflateInit level=%d\n", level);
|
||||||
zwc = ZWRAP_createCCtx();
|
zwc = ZWRAP_createCCtx(strm);
|
||||||
if (zwc == NULL) return Z_MEM_ERROR;
|
if (zwc == NULL) return Z_MEM_ERROR;
|
||||||
|
|
||||||
|
if (level == Z_DEFAULT_COMPRESSION)
|
||||||
|
level = ZWRAP_DEFAULT_CLEVEL;
|
||||||
|
|
||||||
{ size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
|
{ size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
||||||
|
|
||||||
zwc->compressionLevel = level;
|
zwc->compressionLevel = level;
|
||||||
strm->opaque = (voidpf) zwc;
|
strm->state = (struct internal_state*) zwc; /* use state which in not used by user */
|
||||||
strm->total_in = 0;
|
strm->total_in = 0;
|
||||||
strm->total_out = 0;
|
strm->total_out = 0;
|
||||||
return Z_OK;
|
return Z_OK;
|
||||||
@ -146,7 +187,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
|
|||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflateSetDictionary(strm, dictionary, dictLength);
|
return deflateSetDictionary(strm, dictionary, dictLength);
|
||||||
|
|
||||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque;
|
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
|
||||||
LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type);
|
LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type);
|
||||||
{ size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
|
{ size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
||||||
@ -166,7 +207,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
zwc = (ZWRAP_CCtx*) strm->opaque;
|
zwc = (ZWRAP_CCtx*) strm->state;
|
||||||
|
|
||||||
LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||||
if (strm->avail_in > 0) {
|
if (strm->avail_in > 0) {
|
||||||
@ -183,7 +224,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
|||||||
strm->avail_in -= srcSize;
|
strm->avail_in -= srcSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR("Z_FULL_FLUSH is not supported!");
|
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
|
||||||
|
|
||||||
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
|
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
|
||||||
size_t bytesLeft;
|
size_t bytesLeft;
|
||||||
@ -215,7 +256,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm))
|
|||||||
return deflateEnd(strm);
|
return deflateEnd(strm);
|
||||||
}
|
}
|
||||||
LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
||||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque;
|
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
|
||||||
size_t const errorCode = ZWRAP_freeCCtx(zwc);
|
size_t const errorCode = ZWRAP_freeCCtx(zwc);
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
}
|
}
|
||||||
@ -253,25 +294,37 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
|
|||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
ZBUFF_DCtx* zbd;
|
ZBUFF_DCtx* zbd;
|
||||||
char headerBuf[ZSTD_FRAMEHEADERSIZE_MIN];
|
char headerBuf[ZWRAP_HEADERSIZE];
|
||||||
int errorCount;
|
int errorCount;
|
||||||
int requiredHeaderSize;
|
|
||||||
|
|
||||||
/* zlib params */
|
/* zlib params */
|
||||||
int stream_size;
|
int stream_size;
|
||||||
char *version;
|
char *version;
|
||||||
int windowBits;
|
int windowBits;
|
||||||
alloc_func zalloc; /* used to allocate the internal state */
|
ZSTD_customMem customMem;
|
||||||
free_func zfree; /* used to free the internal state */
|
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
|
||||||
voidpf opaque; /* private data object passed to zalloc and zfree */
|
|
||||||
} ZWRAP_DCtx;
|
} ZWRAP_DCtx;
|
||||||
|
|
||||||
|
|
||||||
ZWRAP_DCtx* ZWRAP_createDCtx(void)
|
ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
|
||||||
{
|
{
|
||||||
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx));
|
ZWRAP_DCtx* zwd;
|
||||||
|
|
||||||
|
if (strm->zalloc && strm->zfree) {
|
||||||
|
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
|
||||||
if (zwd==NULL) return NULL;
|
if (zwd==NULL) return NULL;
|
||||||
memset(zwd, 0, sizeof(*zwd));
|
memset(zwd, 0, sizeof(ZWRAP_DCtx));
|
||||||
|
memcpy(&zwd->allocFunc, strm, sizeof(z_stream));
|
||||||
|
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
|
||||||
|
memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx));
|
||||||
|
if (zwd==NULL) return NULL;
|
||||||
|
memset(zwd, 0, sizeof(ZWRAP_DCtx));
|
||||||
|
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||||
|
}
|
||||||
|
|
||||||
return zwd;
|
return zwd;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -279,9 +332,9 @@ ZWRAP_DCtx* ZWRAP_createDCtx(void)
|
|||||||
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
|
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
|
||||||
{
|
{
|
||||||
if (zwd==NULL) return 0; /* support free on null */
|
if (zwd==NULL) return 0; /* support free on null */
|
||||||
if (zwd->version) free(zwd->version);
|
ZBUFF_freeDCtx(zwd->zbd);
|
||||||
if (zwd->zbd) ZBUFF_freeDCtx(zwd->zbd);
|
if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
|
||||||
free(zwd);
|
zwd->customMem.customFree(zwd->customMem.opaque, zwd);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -289,18 +342,16 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
|
|||||||
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
|
||||||
const char *version, int stream_size))
|
const char *version, int stream_size))
|
||||||
{
|
{
|
||||||
ZWRAP_DCtx* zwd = ZWRAP_createDCtx();
|
ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
|
||||||
LOG_WRAPPER("- inflateInit\n");
|
LOG_WRAPPER("- inflateInit\n");
|
||||||
if (zwd == NULL) return Z_MEM_ERROR;
|
if (zwd == NULL) return Z_MEM_ERROR;
|
||||||
|
|
||||||
zwd->stream_size = stream_size;
|
zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
|
||||||
zwd->version = strdup(version);
|
if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; }
|
||||||
zwd->zalloc = strm->zalloc;
|
strcpy(zwd->version, version);
|
||||||
zwd->zfree = strm->zfree;
|
|
||||||
zwd->opaque = strm->opaque;
|
|
||||||
zwd->requiredHeaderSize = ZWRAP_HEADERSIZE;
|
|
||||||
|
|
||||||
strm->state = (struct internal_state*) zwd;
|
zwd->stream_size = stream_size;
|
||||||
|
strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
|
||||||
strm->total_in = 0;
|
strm->total_in = 0;
|
||||||
strm->total_out = 0;
|
strm->total_out = 0;
|
||||||
strm->reserved = 1; /* mark as unknown steam */
|
strm->reserved = 1; /* mark as unknown steam */
|
||||||
@ -360,14 +411,14 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
|||||||
size_t errorCode, dstCapacity, srcSize;
|
size_t errorCode, dstCapacity, srcSize;
|
||||||
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||||
LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||||
while (strm->total_in < ZSTD_FRAMEHEADERSIZE_MIN)
|
if (strm->total_in < ZWRAP_HEADERSIZE)
|
||||||
{
|
{
|
||||||
srcSize = MIN(strm->avail_in, zwd->requiredHeaderSize - strm->total_in);
|
srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in);
|
||||||
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
|
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
|
||||||
strm->total_in += srcSize;
|
strm->total_in += srcSize;
|
||||||
strm->next_in += srcSize;
|
strm->next_in += srcSize;
|
||||||
strm->avail_in -= srcSize;
|
strm->avail_in -= srcSize;
|
||||||
if (strm->total_in < zwd->requiredHeaderSize) return Z_OK;
|
if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK;
|
||||||
|
|
||||||
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
|
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
|
||||||
z_stream strm2;
|
z_stream strm2;
|
||||||
@ -396,21 +447,22 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
|||||||
strm->avail_in = strm2.avail_in;
|
strm->avail_in = strm2.avail_in;
|
||||||
strm->next_out = strm2.next_out;
|
strm->next_out = strm2.next_out;
|
||||||
strm->avail_out = strm2.avail_out;
|
strm->avail_out = strm2.avail_out;
|
||||||
|
|
||||||
strm->reserved = 0; /* mark as zlib stream */
|
strm->reserved = 0; /* mark as zlib stream */
|
||||||
|
errorCode = ZWRAP_freeDCtx(zwd);
|
||||||
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
|
|
||||||
if (flush == Z_INFLATE_SYNC) return inflateSync(strm);
|
if (flush == Z_INFLATE_SYNC) return inflateSync(strm);
|
||||||
return inflate(strm, flush);
|
return inflate(strm, flush);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (zwd->requiredHeaderSize < ZSTD_FRAMEHEADERSIZE_MIN) {
|
zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem);
|
||||||
zwd->requiredHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
if (zwd->zbd == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; }
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
zwd->zbd = ZBUFF_createDCtx();
|
errorCode = ZBUFF_decompressInit(zwd->zbd);
|
||||||
{ size_t const errorCode = ZBUFF_decompressInit(zwd->zbd);
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
|
||||||
|
|
||||||
srcSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
srcSize = ZWRAP_HEADERSIZE;
|
||||||
dstCapacity = 0;
|
dstCapacity = 0;
|
||||||
errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
|
errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
|
||||||
LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
|
LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
|
||||||
@ -419,7 +471,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
|||||||
return Z_MEM_ERROR;
|
return Z_MEM_ERROR;
|
||||||
}
|
}
|
||||||
if (strm->avail_in == 0) return Z_OK;
|
if (strm->avail_in == 0) return Z_OK;
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
srcSize = strm->avail_in;
|
srcSize = strm->avail_in;
|
||||||
@ -447,7 +498,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
|
|||||||
{
|
{
|
||||||
int ret = Z_OK;
|
int ret = Z_OK;
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
ret = inflateEnd(strm);
|
return inflateEnd(strm);
|
||||||
|
|
||||||
LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
||||||
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||||
@ -472,7 +523,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflateCopy(dest, source);
|
return deflateCopy(dest, source);
|
||||||
FINISH_WITH_ERR("deflateCopy is not supported!");
|
FINISH_WITH_ERR(source, "deflateCopy is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -480,7 +531,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflateReset(strm);
|
return deflateReset(strm);
|
||||||
FINISH_WITH_ERR("deflateReset is not supported!");
|
FINISH_WITH_ERR(strm, "deflateReset is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -492,18 +543,20 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflateTune(strm, good_length, max_lazy, nice_length, max_chain);
|
return deflateTune(strm, good_length, max_lazy, nice_length, max_chain);
|
||||||
FINISH_WITH_ERR("deflateTune is not supported!");
|
FINISH_WITH_ERR(strm, "deflateTune is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM >= 0x1260
|
||||||
ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm,
|
||||||
unsigned *pending,
|
unsigned *pending,
|
||||||
int *bits))
|
int *bits))
|
||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflatePending(strm, pending, bits);
|
return deflatePending(strm, pending, bits);
|
||||||
FINISH_WITH_ERR("deflatePending is not supported!");
|
FINISH_WITH_ERR(strm, "deflatePending is not supported!");
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
|
||||||
@ -512,7 +565,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflatePrime(strm, bits, value);
|
return deflatePrime(strm, bits, value);
|
||||||
FINISH_WITH_ERR("deflatePrime is not supported!");
|
FINISH_WITH_ERR(strm, "deflatePrime is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -521,22 +574,23 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return deflateSetHeader(strm, head);
|
return deflateSetHeader(strm, head);
|
||||||
FINISH_WITH_ERR("deflateSetHeader is not supported!");
|
FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* Advanced compression functions */
|
/* Advanced compression functions */
|
||||||
|
#if ZLIB_VERNUM >= 0x1280
|
||||||
ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm,
|
||||||
Bytef *dictionary,
|
Bytef *dictionary,
|
||||||
uInt *dictLength))
|
uInt *dictLength))
|
||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateGetDictionary(strm, dictionary, dictLength);
|
return inflateGetDictionary(strm, dictionary, dictLength);
|
||||||
FINISH_WITH_ERR("inflateGetDictionary is not supported!");
|
FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!");
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
|
ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
|
||||||
@ -544,7 +598,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return inflateCopy(dest, source);
|
return inflateCopy(dest, source);
|
||||||
FINISH_WITH_ERR("inflateCopy is not supported!");
|
FINISH_WITH_ERR(source, "inflateCopy is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -552,17 +606,29 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateReset(strm);
|
return inflateReset(strm);
|
||||||
FINISH_WITH_ERR("inflateReset is not supported!");
|
FINISH_WITH_ERR(strm, "inflateReset is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM >= 0x1240
|
||||||
ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
|
||||||
int windowBits))
|
int windowBits))
|
||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateReset2(strm, windowBits);
|
return inflateReset2(strm, windowBits);
|
||||||
FINISH_WITH_ERR("inflateReset2 is not supported!");
|
FINISH_WITH_ERR(strm, "inflateReset2 is not supported!");
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM >= 0x1240
|
||||||
|
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
|
||||||
|
{
|
||||||
|
if (!strm->reserved)
|
||||||
|
return inflateMark(strm);
|
||||||
|
FINISH_WITH_ERR(strm, "inflateMark is not supported!");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
|
ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
|
||||||
@ -571,15 +637,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflatePrime(strm, bits, value);
|
return inflatePrime(strm, bits, value);
|
||||||
FINISH_WITH_ERR("inflatePrime is not supported!");
|
FINISH_WITH_ERR(strm, "inflatePrime is not supported!");
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
|
|
||||||
{
|
|
||||||
if (!strm->reserved)
|
|
||||||
return inflateMark(strm);
|
|
||||||
FINISH_WITH_ERR("inflateMark is not supported!");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -588,7 +646,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateGetHeader(strm, head);
|
return inflateGetHeader(strm, head);
|
||||||
FINISH_WITH_ERR("inflateGetHeader is not supported!");
|
FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -599,7 +657,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits,
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateBackInit_(strm, windowBits, window, version, stream_size);
|
return inflateBackInit_(strm, windowBits, window, version, stream_size);
|
||||||
FINISH_WITH_ERR("inflateBackInit is not supported!");
|
FINISH_WITH_ERR(strm, "inflateBackInit is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -609,7 +667,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm,
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateBack(strm, in, in_desc, out, out_desc);
|
return inflateBack(strm, in, in_desc, out, out_desc);
|
||||||
FINISH_WITH_ERR("inflateBack is not supported!");
|
FINISH_WITH_ERR(strm, "inflateBack is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -617,7 +675,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
|
|||||||
{
|
{
|
||||||
if (!strm->reserved)
|
if (!strm->reserved)
|
||||||
return inflateBackEnd(strm);
|
return inflateBackEnd(strm);
|
||||||
FINISH_WITH_ERR("inflateBackEnd is not supported!");
|
FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -634,9 +692,9 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
|
|||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return compress(dest, destLen, source, sourceLen);
|
return compress(dest, destLen, source, sourceLen);
|
||||||
|
|
||||||
size_t dstCapacity = *destLen;
|
{ size_t dstCapacity = *destLen;
|
||||||
|
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
|
||||||
LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
|
LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
|
||||||
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1);
|
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
*destLen = errorCode;
|
*destLen = errorCode;
|
||||||
}
|
}
|
||||||
@ -651,8 +709,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen,
|
|||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return compress2(dest, destLen, source, sourceLen, level);
|
return compress2(dest, destLen, source, sourceLen, level);
|
||||||
|
|
||||||
size_t dstCapacity = *destLen;
|
{ size_t dstCapacity = *destLen;
|
||||||
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
|
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
*destLen = errorCode;
|
*destLen = errorCode;
|
||||||
}
|
}
|
||||||
@ -676,8 +734,8 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen,
|
|||||||
// if (!g_useZSTD)
|
// if (!g_useZSTD)
|
||||||
return uncompress(dest, destLen, source, sourceLen);
|
return uncompress(dest, destLen, source, sourceLen);
|
||||||
|
|
||||||
size_t dstCapacity = *destLen;
|
{ size_t dstCapacity = *destLen;
|
||||||
{ size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
|
size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
|
||||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||||
*destLen = errorCode;
|
*destLen = errorCode;
|
||||||
}
|
}
|
||||||
@ -703,19 +761,45 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode))
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM >= 0x1240
|
||||||
ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size))
|
ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size))
|
||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzbuffer(file, size);
|
return gzbuffer(file, size);
|
||||||
FINISH_WITH_ERR("gzbuffer is not supported!");
|
FINISH_WITH_GZ_ERR("gzbuffer is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
|
||||||
|
{
|
||||||
|
if (!g_useZSTD)
|
||||||
|
return gzoffset(file);
|
||||||
|
FINISH_WITH_GZ_ERR("gzoffset is not supported!");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
|
||||||
|
{
|
||||||
|
if (!g_useZSTD)
|
||||||
|
return gzclose_r(file);
|
||||||
|
FINISH_WITH_GZ_ERR("gzclose_r is not supported!");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
|
||||||
|
{
|
||||||
|
if (!g_useZSTD)
|
||||||
|
return gzclose_w(file);
|
||||||
|
FINISH_WITH_GZ_ERR("gzclose_w is not supported!");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy))
|
ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy))
|
||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzsetparams(file, level, strategy);
|
return gzsetparams(file, level, strategy);
|
||||||
FINISH_WITH_ERR("gzsetparams is not supported!");
|
FINISH_WITH_GZ_ERR("gzsetparams is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -723,7 +807,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzread(file, buf, len);
|
return gzread(file, buf, len);
|
||||||
FINISH_WITH_ERR("gzread is not supported!");
|
FINISH_WITH_GZ_ERR("gzread is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -732,11 +816,15 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file,
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzwrite(file, buf, len);
|
return gzwrite(file, buf, len);
|
||||||
FINISH_WITH_ERR("gzwrite is not supported!");
|
FINISH_WITH_GZ_ERR("gzwrite is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM >= 0x1260
|
||||||
ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
|
ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
|
||||||
|
#else
|
||||||
|
ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...))
|
||||||
|
#endif
|
||||||
{
|
{
|
||||||
if (!g_useZSTD) {
|
if (!g_useZSTD) {
|
||||||
int ret;
|
int ret;
|
||||||
@ -750,7 +838,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
|
|||||||
// printf("gzprintf ret=%d\n", ret);
|
// printf("gzprintf ret=%d\n", ret);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
FINISH_WITH_ERR("gzprintf is not supported!");
|
FINISH_WITH_GZ_ERR("gzprintf is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -758,7 +846,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzputs(file, s);
|
return gzputs(file, s);
|
||||||
FINISH_WITH_ERR("gzputs is not supported!");
|
FINISH_WITH_GZ_ERR("gzputs is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -774,15 +862,19 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzputc(file, c);
|
return gzputc(file, c);
|
||||||
FINISH_WITH_ERR("gzputc is not supported!");
|
FINISH_WITH_GZ_ERR("gzputc is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#if ZLIB_VERNUM == 0x1260
|
||||||
|
ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file))
|
||||||
|
#else
|
||||||
ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file))
|
ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file))
|
||||||
|
#endif
|
||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzgetc(file);
|
return gzgetc(file);
|
||||||
FINISH_WITH_ERR("gzgetc is not supported!");
|
FINISH_WITH_GZ_ERR("gzgetc is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -790,7 +882,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzungetc(c, file);
|
return gzungetc(c, file);
|
||||||
FINISH_WITH_ERR("gzungetc is not supported!");
|
FINISH_WITH_GZ_ERR("gzungetc is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -798,7 +890,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzflush(file, flush);
|
return gzflush(file, flush);
|
||||||
FINISH_WITH_ERR("gzflush is not supported!");
|
FINISH_WITH_GZ_ERR("gzflush is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -806,7 +898,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzseek(file, offset, whence);
|
return gzseek(file, offset, whence);
|
||||||
FINISH_WITH_ERR("gzseek is not supported!");
|
FINISH_WITH_GZ_ERR("gzseek is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -814,7 +906,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzrewind(file);
|
return gzrewind(file);
|
||||||
FINISH_WITH_ERR("gzrewind is not supported!");
|
FINISH_WITH_GZ_ERR("gzrewind is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -822,15 +914,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gztell(file);
|
return gztell(file);
|
||||||
FINISH_WITH_ERR("gztell is not supported!");
|
FINISH_WITH_GZ_ERR("gztell is not supported!");
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
|
|
||||||
{
|
|
||||||
if (!g_useZSTD)
|
|
||||||
return gzoffset(file);
|
|
||||||
FINISH_WITH_ERR("gzoffset is not supported!");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -838,7 +922,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzeof(file);
|
return gzeof(file);
|
||||||
FINISH_WITH_ERR("gzeof is not supported!");
|
FINISH_WITH_GZ_ERR("gzeof is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -846,7 +930,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzdirect(file);
|
return gzdirect(file);
|
||||||
FINISH_WITH_ERR("gzdirect is not supported!");
|
FINISH_WITH_GZ_ERR("gzdirect is not supported!");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -854,23 +938,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file))
|
|||||||
{
|
{
|
||||||
if (!g_useZSTD)
|
if (!g_useZSTD)
|
||||||
return gzclose(file);
|
return gzclose(file);
|
||||||
FINISH_WITH_ERR("gzclose is not supported!");
|
FINISH_WITH_GZ_ERR("gzclose is not supported!");
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
|
|
||||||
{
|
|
||||||
if (!g_useZSTD)
|
|
||||||
return gzclose_r(file);
|
|
||||||
FINISH_WITH_ERR("gzclose_r is not supported!");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
|
|
||||||
{
|
|
||||||
if (!g_useZSTD)
|
|
||||||
return gzclose_w(file);
|
|
||||||
FINISH_WITH_ERR("gzclose_w is not supported!");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -40,10 +40,17 @@ extern "C" {
|
|||||||
#define Z_PREFIX
|
#define Z_PREFIX
|
||||||
#include <zlib.h>
|
#include <zlib.h>
|
||||||
|
|
||||||
|
#if !defined(z_const)
|
||||||
|
#if ZLIB_VERNUM >= 0x1260
|
||||||
|
#define z_const const
|
||||||
|
#else
|
||||||
|
#define z_const
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
void useZSTD(int turn_on);
|
void useZSTD(int turn_on);
|
||||||
int isUsingZSTD();
|
int isUsingZSTD(void);
|
||||||
const char * zstdVersion();
|
const char * zstdVersion(void);
|
||||||
|
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user