Merge pull request #200 from Cyan4973/dev070

Dev070
This commit is contained in:
Yann Collet 2016-06-06 12:12:28 +02:00
commit c44785be57
75 changed files with 3037 additions and 2783 deletions

1
.gitattributes vendored
View File

@ -12,6 +12,7 @@
# Visual Studio # Visual Studio
*.sln text eol=crlf *.sln text eol=crlf
*.vcxproj* text eol=crlf *.vcxproj* text eol=crlf
*.vcproj* text eol=crlf
*.suo binary *.suo binary
*.rc binary *.rc binary

4
.gitignore vendored
View File

@ -31,7 +31,7 @@ projects/VS2015
.directory .directory
_codelite/ _codelite/
_zstdbench/ _zstdbench/
zlib_wrapper/ .clang_complete
# CMake # CMake
contrib/cmake/ projects/cmake/

View File

@ -1,34 +1,90 @@
language: c language: c
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq clang
- sudo apt-get install -qq g++-multilib
- sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind
env:
- ZSTD_TRAVIS_CI_ENV=travis-install
- ZSTD_TRAVIS_CI_ENV=cmaketest
- ZSTD_TRAVIS_CI_ENV=clangtest
- ZSTD_TRAVIS_CI_ENV=gpptest
- ZSTD_TRAVIS_CI_ENV=gnu90test
- ZSTD_TRAVIS_CI_ENV=c99test
- ZSTD_TRAVIS_CI_ENV=gnu99test
- ZSTD_TRAVIS_CI_ENV=armtest-w-install
- ZSTD_TRAVIS_CI_ENV=test
- ZSTD_TRAVIS_CI_ENV="-C programs test32"
- ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
- ZSTD_TRAVIS_CI_ENV="-C versionsTest"
- ZSTD_TRAVIS_CI_ENV=usan
- ZSTD_TRAVIS_CI_ENV=asan
- ZSTD_TRAVIS_CI_ENV=asan32
- ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"
compiler: gcc compiler: gcc
script:
- make $ZSTD_TRAVIS_CI_ENV
matrix: matrix:
fast_finish: true fast_finish: true
include:
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
# - os: linux
# sudo: required
# env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
# OS X Mavericks
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
- os: osx
env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
exclude:
- compiler: gcc
before_install:
- set -e
- |
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
fi
fi
script:
- make $MAKE_PARAM

129
Makefile
View File

@ -32,6 +32,7 @@
PRGDIR = programs PRGDIR = programs
ZSTDDIR = lib ZSTDDIR = lib
ZWRAPDIR = zlibWrapper
# Define nul output # Define nul output
ifneq (,$(filter Windows%,$(OS))) ifneq (,$(filter Windows%,$(OS)))
@ -40,7 +41,7 @@ else
VOID = /dev/null VOID = /dev/null
endif endif
.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan .PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
default: zstdprogram default: zstdprogram
@ -51,19 +52,25 @@ all:
zstdprogram: zstdprogram:
$(MAKE) -C $(PRGDIR) $(MAKE) -C $(PRGDIR)
zlibwrapper:
$(MAKE) -C $(ZSTDDIR) all
$(MAKE) -C $(ZWRAPDIR) all
test: test:
$(MAKE) -C $(PRGDIR) $@ $(MAKE) -C $(PRGDIR) $@
clean: clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
@echo Cleaning completed @echo Cleaning completed
#------------------------------------------------------------------------ #------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
HOST_OS = POSIX
install: install:
$(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ $(MAKE) -C $(PRGDIR) $@
@ -75,15 +82,80 @@ uninstall:
travis-install: travis-install:
$(MAKE) install PREFIX=~/install_test_dir $(MAKE) install PREFIX=~/install_test_dir
cmaketest: gpptest: clean
cd projects/cmake/build ; cmake .. ; $(MAKE) $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
clangtest: clean clangtest: clean
clang -v clang -v
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
gpptest: clean armtest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" $(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
arminstall: clean
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
# for Travis CI
armtest-w-install: clean arminstall armtest
ppctest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
ppcinstall: clean
# sudo apt-get update -y -q
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
# for Travis CI
ppctest-w-install: clean ppcinstall ppctest
ppc64test: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
ppc64install: clean
sudo apt-get update -y -q
sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
update-binfmts --displ
ppc64test-w-install: clean ppc64install ppc64test
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
endif
#------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
#------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
cmaketest:
cmake --version
rm -rf projects/cmake/build
mkdir projects/cmake/build
cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
c90test: clean c90test: clean
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
@ -108,49 +180,4 @@ bmix32test: clean
bmi32test: clean bmi32test: clean
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
armtest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
arminstall: clean
sudo apt-get install -q qemu
sudo apt-get install -q binfmt-support
sudo apt-get install -q qemu-user-static
sudo apt-get install -q gcc-arm-linux-gnueabi
# for Travis CI
armtest-w-install: clean arminstall armtest
ppctest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
ppcinstall: clean
sudo apt-get install -q qemu
sudo apt-get install -q binfmt-support
sudo apt-get install -q qemu-user-static
sudo apt-get update -q
sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available)
# for Travis CI
ppctest-w-install: clean ppcinstall ppctest
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
endif endif

10
NEWS
View File

@ -1,8 +1,12 @@
v0.6.2 v0.7.0
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Support for Sparse File-systems (do not use space for zero-filled sectors)
New : Support pass-through mode, when using `-df` New : Frame checksum support
New : Support for custom malloc/free functions New : Support pass-through mode (when using `-df`)
New : API : dictionary files from custom content, by Giuseppe Ottaviano
New : API support for custom malloc/free functions
New : controllable Dictionary ID
New : Support for skippable frames
v0.6.1 v0.6.1
New : zlib wrapper API, thanks to Przemyslaw Skibinski New : zlib wrapper API, thanks to Przemyslaw Skibinski

View File

@ -1,24 +1,24 @@
version: 1.0.{build} version: 1.0.{build}
environment: environment:
matrix: matrix:
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
- COMPILER: "gcc" - COMPILER: "gcc"
MAKE_PARAMS: "test" MAKE_PARAMS: "test"
PLATFORM: "mingw64" PLATFORM: "mingw64"
- COMPILER: "gcc" - COMPILER: "gcc"
MAKE_PARAMS: "test" MAKE_PARAMS: "test"
PLATFORM: "mingw32" PLATFORM: "mingw32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
install: install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
@ -80,7 +80,7 @@ build_script:
ECHO *** && ECHO *** &&
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
) )
test_script: test_script:
@ -91,12 +91,11 @@ test_script:
) )
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
CD programs && CD programs &&
SET ZSTD=..\projects\zstd.exe && SET ZSTD=./zstd.exe &&
SET DATAGEN=..\projects\datagen.exe && sh -e playTests.sh --test-large-data &&
CALL tests\playTests.bat --test-large-data && fullbench.exe -i1 &&
fullbench.exe -i1 -P0 &&
CD .. && CD .. &&
projects\fullbench.exe -i1 &&
projects\fullbench.exe -i1 -P0 &&
projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&

View File

@ -43,24 +43,21 @@ VERSION?= $(LIBVER)
DESTDIR?= DESTDIR?=
PREFIX ?= /usr/local PREFIX ?= /usr/local
CPPFLAGS= -I./common
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
LIBDIR ?= $(PREFIX)/lib LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include INCLUDEDIR=$(PREFIX)/include
ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c CFLAGS ?= -O3
ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
else else
ZSTD_FILES+= $(ZSTD_LEGACY) ZSTD_FILES+= legacy/*.c
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1 CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
endif endif

View File

@ -12,13 +12,10 @@ To build the zstd library the following files are required:
- [common/error_public.h](common/error_public.h) - [common/error_public.h](common/error_public.h)
- common/fse.h - common/fse.h
- common/fse_decompress.c - common/fse_decompress.c
- common/fse_static.h
- common/huf.h - common/huf.h
- common/huf_static.h
- [common/mem.h](common/mem.h) - [common/mem.h](common/mem.h)
- [common/zstd.h] - [common/zstd.h]
- common/zstd_internal.h - common/zstd_internal.h
- common/zstd_static.h
- compress/fse_compress.c - compress/fse_compress.c
- compress/huf_compress.c - compress/huf_compress.c
- compress/zstd_compress.c - compress/zstd_compress.c
@ -27,12 +24,10 @@ To build the zstd library the following files are required:
- decompress/zstd_decompress.c - decompress/zstd_decompress.c
Stable API is exposed in [common/zstd.h]. Stable API is exposed in [common/zstd.h].
Advanced and experimental API is exposed in [common/zstd_static.h]. Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`.
API elements of [common/zstd_static.h] should be used with static linking only, Never use them with a dynamic library, as their definition may change in future versions.
as their definition may change in future version of the library.
[common/zstd.h]: common/zstd.h [common/zstd.h]: common/zstd.h
[common/zstd_static.h]: common/zstd_static.h
#### Separate compressor and decompressor #### Separate compressor and decompressor
@ -47,7 +42,6 @@ This complementary API makes streaming integration easier.
It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) : It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
- common/zbuff.h - common/zbuff.h
- common/zbuff_static.h
- compress/zbuff_compress.c - compress/zbuff_compress.c
- decompress/zbuff_decompress.c - decompress/zbuff_decompress.c
@ -59,7 +53,6 @@ To create dictionaries from training sets :
- dictBuilder/divsufsort.h - dictBuilder/divsufsort.h
- dictBuilder/zdict.c - dictBuilder/zdict.c
- dictBuilder/zdict.h - dictBuilder/zdict.h
- dictBuilder/zdict_static.h
#### Miscellaneous #### Miscellaneous
@ -68,4 +61,3 @@ The other files are not source code. There are :
- LICENSE : contains the BSD license text - LICENSE : contains the BSD license text
- Makefile : script to compile or install zstd library (static or dynamic) - Makefile : script to compile or install zstd library (static or dynamic)
- libzstd.pc.in : for pkg-config (make install) - libzstd.pc.in : for pkg-config (make install)

View File

@ -35,12 +35,12 @@
/* ************************************* /* *************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include <stdlib.h>
#include "mem.h" #include "mem.h"
#include "fse_static.h" /* FSE_MIN_TABLELOG */ #include "error_private.h" /* ERR_*, ERROR */
#include "error_private.h" #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ #include "fse.h" /* FSE_isError, FSE_getErrorName */
#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ #define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
#include "huf.h" /* HUF_isError, HUF_getErrorName */
@ -156,3 +156,76 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
return ip-istart; return ip-istart;
} }
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() .
*/
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t oSize;
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
} }
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
return iSize+1;
}

View File

@ -102,10 +102,12 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect"; case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected"; case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode): case PREFIX(maxCode):
default: return notErrorCode; default: return notErrorCode;
} }

View File

@ -54,10 +54,12 @@ typedef enum {
ZSTD_error_dstSize_tooSmall, ZSTD_error_dstSize_tooSmall,
ZSTD_error_srcSize_wrong, ZSTD_error_srcSize_wrong,
ZSTD_error_corruption_detected, ZSTD_error_corruption_detected,
ZSTD_error_checksum_wrong,
ZSTD_error_tableLog_tooLarge, ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge, ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall, ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted, ZSTD_error_dictionary_corrupted,
ZSTD_error_dictionary_wrong,
ZSTD_error_maxCode ZSTD_error_maxCode
} ZSTD_ErrorCode; } ZSTD_ErrorCode;

View File

@ -272,6 +272,355 @@ If there is an error, the function will return an error code, which can be teste
*/ */
#ifdef FSE_STATIC_LINKING_ONLY
/* *** Dependency *** */
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/* *****************************************
* FSE advanced API
*******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
If none of these solutions is applicable, include "fse.c" directly.
*/
typedef struct
{
ptrdiff_t value;
const void* stateTable;
const void* symbolTT;
unsigned stateLog;
} FSE_CState_t;
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/**<
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
So the first symbol you will encode is the last you will decode, like a LIFO stack.
You will need a few variables to track your CStream. They are :
FSE_CTable ct; // Provided by FSE_buildCTable()
BIT_CStream_t bitStream; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
At any time, you can also add any bit sequence.
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
BIT_addBits(&bitStream, bitField, nbBits);
The above methods don't commit data to memory, they just store it into local register, for speed.
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
Writing data to memory is a manual operation, performed by the flushBits function.
BIT_flushBits(&bitStream);
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
Finally, you must close the bitStream.
The function returns the size of CStream in bytes.
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = BIT_closeCStream(&bitStream);
*/
/* *****************************************
* FSE symbol decompression API
*******************************************/
typedef struct
{
size_t state;
const void* table; /* precise table may vary, depending on U16 */
} FSE_DState_t;
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/**<
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
You will need a few variables to track your bitStream. They are :
BIT_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
You should then retrieve your initial state(s)
(in reverse flushing order if you have several ones) :
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
Note : maximum allowed nbBits is 25, for 32-bits compatibility
size_t bitField = BIT_readBits(&DStream, nbBits);
All above operations only read from local register (which size depends on size_t).
Refueling the register from memory is manually performed by the reload method.
endSignal = FSE_reloadDStream(&DStream);
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
BIT_DStream_unfinished : there is still some data left into the DStream.
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
BIT_endOfDStream(&DStream);
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
/* *****************************************
* FSE unsafe API
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct {
int deltaFindState;
U32 deltaNbBits;
} FSE_symbolCompressionTransform; /* total 8 bytes */
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
/*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
{
FSE_initCState(statePtr, ct);
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
{
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/*<===== Decompression =====>*/
typedef struct {
U16 tableLog;
U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */
typedef struct
{
unsigned short newState;
unsigned char symbol;
unsigned char nbBits;
} FSE_decode_t; /* size == U32 */
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/* **************************************************************
* Tuning parameters
****************************************************************/
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
#define FSE_MIN_TABLELOG 5
#define FSE_TABLELOG_ABSOLUTE_MAX 15
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
#endif
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
#endif /* FSE_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

View File

@ -58,7 +58,8 @@
#include <string.h> /* memcpy, memset */ #include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */ #include <stdio.h> /* printf (debug) */
#include "bitstream.h" #include "bitstream.h"
#include "fse_static.h" #define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* ************************************************************** /* **************************************************************

View File

@ -1,392 +0,0 @@
/* ******************************************************************
FSE : Finite State Entropy coder
header file for static linking (only)
Copyright (C) 2013-2015, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
#ifndef FSE_STATIC_H
#define FSE_STATIC_H
#if defined (__cplusplus)
extern "C" {
#endif
/* *****************************************
* Dependencies
*******************************************/
#include "fse.h"
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/* *****************************************
* FSE advanced API
*******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
If none of these solutions is applicable, include "fse.c" directly.
*/
typedef struct
{
ptrdiff_t value;
const void* stateTable;
const void* symbolTT;
unsigned stateLog;
} FSE_CState_t;
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/**<
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
So the first symbol you will encode is the last you will decode, like a LIFO stack.
You will need a few variables to track your CStream. They are :
FSE_CTable ct; // Provided by FSE_buildCTable()
BIT_CStream_t bitStream; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
At any time, you can also add any bit sequence.
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
BIT_addBits(&bitStream, bitField, nbBits);
The above methods don't commit data to memory, they just store it into local register, for speed.
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
Writing data to memory is a manual operation, performed by the flushBits function.
BIT_flushBits(&bitStream);
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
Finally, you must close the bitStream.
The function returns the size of CStream in bytes.
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = BIT_closeCStream(&bitStream);
*/
/* *****************************************
* FSE symbol decompression API
*******************************************/
typedef struct
{
size_t state;
const void* table; /* precise table may vary, depending on U16 */
} FSE_DState_t;
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/**<
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
You will need a few variables to track your bitStream. They are :
BIT_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
You should then retrieve your initial state(s)
(in reverse flushing order if you have several ones) :
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
Note : maximum allowed nbBits is 25, for 32-bits compatibility
size_t bitField = BIT_readBits(&DStream, nbBits);
All above operations only read from local register (which size depends on size_t).
Refueling the register from memory is manually performed by the reload method.
endSignal = FSE_reloadDStream(&DStream);
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
BIT_DStream_unfinished : there is still some data left into the DStream.
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
BIT_endOfDStream(&DStream);
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
/* *****************************************
* FSE unsafe API
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct {
int deltaFindState;
U32 deltaNbBits;
} FSE_symbolCompressionTransform; /* total 8 bytes */
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
{
FSE_initCState(statePtr, ct);
{
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
{
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/*<===== Decompression =====>*/
typedef struct {
U16 tableLog;
U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */
typedef struct
{
unsigned short newState;
unsigned char symbol;
unsigned char nbBits;
} FSE_decode_t; /* size == U32 */
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/* **************************************************************
* Tuning parameters
****************************************************************/
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
#define FSE_MIN_TABLELOG 5
#define FSE_TABLELOG_ABSOLUTE_MAX 15
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
#endif
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
#if defined (__cplusplus)
}
#endif
#endif /* FSE_STATIC_H */

View File

@ -39,13 +39,11 @@ extern "C" {
#endif #endif
/* **************************************** /* *** Dependencies *** */
* Dependency
******************************************/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/* **************************************** /*-****************************************
* HUF simple functions * HUF simple functions
******************************************/ ******************************************/
size_t HUF_compress(void* dst, size_t dstCapacity, size_t HUF_compress(void* dst, size_t dstCapacity,
@ -54,17 +52,17 @@ size_t HUF_decompress(void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize); const void* cSrc, size_t cSrcSize);
/* /*
HUF_compress() : HUF_compress() :
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize). 'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
Note : srcSize must be <= 128 KB Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB
@return : size of compressed data (<= dstCapacity) @return : size of compressed data (<= `dstCapacity`)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression). if return == 1, srcData is a single repeated byte symbol (RLE compression).
if HUF_isError(return), compression failed (more details using HUF_getErrorName()) if HUF_isError(return), compression failed (more details using HUF_getErrorName())
HUF_decompress() : HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated destination buffer 'dst', of size 'dstSize'. into already allocated buffer 'dst', of minimum size 'dstSize'.
`dstSize` : must be the **exact** size of original (uncompressed) data. `dstSize` : must be the **exact** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
@ -78,19 +76,137 @@ HUF_decompress() :
* Tool functions * Tool functions
******************************************/ ******************************************/
#define HUF_BLOCKSIZE_MAX (128 * 1024) #define HUF_BLOCKSIZE_MAX (128 * 1024)
size_t HUF_compressBound(size_t size); /**< maximum compressed size */ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
/* Error Management */ /* Error Management */
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
/* **************************************** /* *** Advanced function *** */
* Advanced functions
******************************************/ /** HUF_compress2() :
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
#ifdef HUF_STATIC_LINKING_ONLY
/* *** Dependencies *** */
#include "mem.h" /* U32 */
/* *** Constants *** */
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
# error "HUF_TABLELOG_MAX is too large !"
#endif
/* ****************************************
* Static allocation
******************************************/
/* HUF buffer bounds */
#define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* static allocation of HUF's Compression Table */
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
U32 name##hb[maxSymbolValue+1]; \
void* name##hv = &(name##hb); \
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
/* ****************************************
* HUF detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. (optional) refine tableLog using HUF_optimalTableLog()
3. build Huffman table from count using HUF_buildCTable()
4. save Huffman table to memory buffer using HUF_writeCTable()
5. encode the data stream using HUF_compress4X_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
/** HUF_readCTable() :
* Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
#endif /* HUF_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

View File

@ -1,232 +0,0 @@
/* ******************************************************************
Huffman codec, part of New Generation Entropy library
header file, for static linking only
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef HUF_STATIC_H
#define HUF_STATIC_H
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependency
******************************************/
#include "huf.h"
#include "fse.h"
#include "bitstream.h"
/* ****************************************
* Static allocation
******************************************/
/* HUF buffer bounds */
#define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* static allocation of HUF's Compression Table */
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
U32 name##hb[maxSymbolValue+1]; \
void* name##hv = &(name##hb); \
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
/* ****************************************
* HUF detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. (optional) refine tableLog using HUF_optimalTableLog()
3. build Huffman table from count using HUF_buildCTable()
4. save Huffman table to memory buffer using HUF_writeCTable()
5. encode the data stream using HUF_compress4X_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
/* **************************************************************
* Constants
****************************************************************/
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
# error "HUF_TABLELOG_MAX is too large !"
#endif
/* **************************************************************
* Needed by zstd in both compression and decompression
****************************************************************/
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src`
*/
MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t oSize;
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
} }
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
return iSize+1;
}
#if defined (__cplusplus)
}
#endif
#endif /* HUF_STATIC_H */

View File

@ -28,8 +28,8 @@
You can contact the author at : You can contact the author at :
- zstd homepage : http://www.zstd.net/ - zstd homepage : http://www.zstd.net/
*/ */
#ifndef ZSTD_BUFFERED_H #ifndef ZSTD_BUFFERED_H_23987
#define ZSTD_BUFFERED_H #define ZSTD_BUFFERED_H_23987
#if defined (__cplusplus) #if defined (__cplusplus)
extern "C" { extern "C" {
@ -160,8 +160,40 @@ ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
#ifdef ZBUFF_STATIC_LINKING_ONLY
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
* ==================================================================================== */
/*--- Dependency ---*/
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "zstd.h"
/*--- External memory ---*/
/*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/*--- Advanced Streaming function ---*/
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize);
#endif /* ZBUFF_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
#endif /* ZSTD_BUFFERED_H */ #endif /* ZSTD_BUFFERED_H_23987 */

View File

@ -1,84 +0,0 @@
/*
zstd - buffered version of compression library
experimental complementary API, for static linking only
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : http://www.zstd.net
*/
#ifndef ZSTD_BUFFERED_STATIC_H
#define ZSTD_BUFFERED_STATIC_H
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include "zstd_static.h" /* ZSTD_parameters */
#include "zbuff.h"
#define ZBUFF_MIN(a,b) ((a)<(b) ? (a) : (b))
/*-*************************************
* Advanced functions
***************************************/
/*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/* *************************************
* Advanced Streaming functions
***************************************/
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize);
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t length = ZBUFF_MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_BUFFERED_STATIC_H */

View File

@ -29,8 +29,8 @@
You can contact the author at : You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd - zstd source repository : https://github.com/Cyan4973/zstd
*/ */
#ifndef ZSTD_H #ifndef ZSTD_H_235446
#define ZSTD_H #define ZSTD_H_235446
#if defined (__cplusplus) #if defined (__cplusplus)
extern "C" { extern "C" {
@ -60,8 +60,8 @@ extern "C" {
* Version * Version
***************************************/ ***************************************/
#define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 6 #define ZSTD_VERSION_MINOR 7
#define ZSTD_VERSION_RELEASE 2 #define ZSTD_VERSION_RELEASE 0
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str #define ZSTD_QUOTE(str) #str
@ -148,8 +148,267 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict,size_t dictSize); const void* dict,size_t dictSize);
#ifdef ZSTD_STATIC_LINKING_ONLY
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
* ==================================================================================== */
/*--- Dependency ---*/
#include "mem.h"
/*--- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN 4
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG3_MAX 17
#define ZSTD_HASHLOG3_MIN 15
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 3
#define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_min = 5;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
/*--- Types ---*/
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
typedef struct {
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == faster, more memory */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy;
} ZSTD_compressionParameters;
typedef struct {
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
ZSTD_compressionParameters cParams;
ZSTD_frameParameters fParams;
} ZSTD_parameters;
/* custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/*-*************************************
* Advanced functions
***************************************/
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
* `srcSize` value is optional, select 0 if not known */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
/*! ZSTD_checkParams() :
* Ensure param values remain within authorized range */
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
/*! ZSTD_adjustParams() :
* optimize params for a given `srcSize` and `dictSize`.
* both values are optional, select `0` if unknown. */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() :
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params);
/*! ZSTD_compress_usingPreparedDCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*- Advanced Decompression functions -*/
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/* **************************************
* Streaming functions (direct mode)
****************************************/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
/*
Streaming compression, synchronous mode (bufferless)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
or ZSTD_compressBegin_advanced(), for finer parameter control.
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
Then, consume your input using ZSTD_compressContinue().
The interface is synchronous, so all input will be consumed and produce a compressed output.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
Without the epilogue, frames will be considered incomplete by decoder.
You can then reuse ZSTD_CCtx to compress some new frame.
*/
typedef struct {
U64 frameContentSize;
U32 windowSize;
U32 dictID;
U32 checksumFlag;
} ZSTD_frameParams;
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
Streaming decompression, direct mode (bufferless)
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
It can provide the minimum size of rolling buffer required to properly decompress data,
and optionally the final size of uncompressed content.
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
Frame parameters are extracted from the beginning of compressed frame.
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError()
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following:
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_decompressContinue() always returns 0.
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
/* **************************************
* Block functions
****************************************/
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
A few rules to respect :
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
- Compressing or decompressing requires a context structure
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
- It is necessary to init context before starting
+ compression : ZSTD_compressBegin()
+ decompression : ZSTD_decompressBegin()
+ variants _usingDict() are also allowed
+ copyCCtx() and copyDCtx() work too
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
In which case, nothing is produced into `dst`.
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
*/
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-*************************************
* Error management
***************************************/
#include "error_public.h"
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#endif /* ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
#endif /* ZSTD_H */ #endif /* ZSTD_H_235446 */

View File

@ -33,8 +33,10 @@
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include <stdlib.h> /* malloc */
#include "error_private.h" #include "error_private.h"
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
@ -70,3 +72,20 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
return address;
}
void ZSTD_defaultFreeFunction(void* opaque, void* address)
{
(void)opaque;
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
free(address);
}

View File

@ -37,7 +37,8 @@
***************************************/ ***************************************/
#include "mem.h" #include "mem.h"
#include "error_private.h" #include "error_private.h"
#include "zstd_static.h" #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
/*-************************************* /*-*************************************
@ -63,7 +64,7 @@
#endif #endif
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A436 #define ZSTD_DICT_MAGIC 0xEC30A437
#define ZSTD_REP_NUM 3 #define ZSTD_REP_NUM 3
#define ZSTD_REP_INIT ZSTD_REP_NUM #define ZSTD_REP_INIT ZSTD_REP_NUM
@ -80,10 +81,11 @@
#define BIT1 2 #define BIT1 2
#define BIT0 1 #define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
@ -160,28 +162,6 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
while (op < oend); while (op < oend);
} }
MEM_STATIC unsigned ZSTD_highbit(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-******************************************* /*-*******************************************
* Private interfaces * Private interfaces
@ -209,7 +189,7 @@ typedef struct {
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif // #if ZSTD_OPT_DEBUG == 3 #endif /* #if ZSTD_OPT_DEBUG == 3 */
typedef struct { typedef struct {
void* buffer; void* buffer;
@ -252,6 +232,11 @@ typedef struct {
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
/* custom memory allocation functions */
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
void ZSTD_defaultFreeFunction(void* opaque, void* address);
static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
#endif /* ZSTD_CCOMMON_H_MODULE */ #endif /* ZSTD_CCOMMON_H_MODULE */

View File

@ -1,284 +0,0 @@
/*
zstd - standard compression library
Header File for static linking only
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : http://www.zstd.net
*/
#ifndef ZSTD_STATIC_H
#define ZSTD_STATIC_H
/* The prototypes defined within this file are considered experimental.
* They should not be used in the context DLL as they may change in the future.
* Prefer static linking if you need them, to control breaking version changes issues.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Dependencies
***************************************/
#include "zstd.h"
#include "mem.h"
/*-*************************************
* Constants
***************************************/
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
/*-*************************************
* Types
***************************************/
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN 4
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG3_MAX 17
#define ZSTD_HASHLOG3_MIN 15
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 3
#define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MAX 999
/* from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;
typedef struct {
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == faster, more memory */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy;
} ZSTD_compressionParameters;
typedef struct {
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
} ZSTD_frameParameters;
typedef struct {
ZSTD_compressionParameters cParams;
ZSTD_frameParameters fParams;
} ZSTD_parameters;
typedef void* (*ZSTD_allocFunction) (size_t size);
typedef void (*ZSTD_freeFunction) (void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem;
/*-*************************************
* Advanced functions
***************************************/
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
* `srcSize` value is optional, select 0 if not known */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
/*! ZSTD_checkParams() :
* Ensure param values remain within authorized range */
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
/*! ZSTD_adjustParams() :
* optimize params for a given `srcSize` and `dictSize`.
* both values are optional, select `0` if unknown. */
ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() :
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params);
/*! ZSTD_compress_usingPreparedDCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*- Advanced Decompression functions -*/
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/* **************************************
* Streaming functions (direct mode)
****************************************/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
/*
Streaming compression, synchronous mode (bufferless)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
or ZSTD_compressBegin_advanced(), for finer parameter control.
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
Then, consume your input using ZSTD_compressContinue().
The interface is synchronous, so all input will be consumed and produce a compressed output.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
Without the epilogue, frames will be considered incomplete by decoder.
You can then reuse ZSTD_CCtx to compress some new frame.
*/
typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_min = 5;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
Streaming decompression, direct mode (bufferless)
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
It can provide the minimum size of rolling buffer required to properly decompress data,
and optionally the final size of uncompressed content.
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
Frame parameters are extracted from the beginning of compressed frame.
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError()
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
*/
/* **************************************
* Block functions
****************************************/
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
A few rules to respect :
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
- Compressing or decompressing requires a context structure
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
- It is necessary to init context before starting
+ compression : ZSTD_compressBegin()
+ decompression : ZSTD_decompressBegin()
+ variants _usingDict() are also allowed
+ copyCCtx() and copyDCtx() work too
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
In which case, nothing is produced into `dst`.
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
*/
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-*************************************
* Error management
***************************************/
#include "error_public.h"
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_STATIC_H */

View File

@ -57,7 +57,8 @@
#include <string.h> /* memcpy, memset */ #include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */ #include <stdio.h> /* printf (debug) */
#include "bitstream.h" #include "bitstream.h"
#include "fse_static.h" #define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* ************************************************************** /* **************************************************************

View File

@ -59,12 +59,13 @@
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */ #include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */ #include <stdio.h> /* printf (debug) */
#include "huf_static.h"
#include "bitstream.h" #include "bitstream.h"
#include "fse_static.h" /* header compression */ #define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* ************************************************************** /* **************************************************************

View File

@ -35,9 +35,9 @@
***************************************/ ***************************************/
#include <stdlib.h> #include <stdlib.h>
#include "error_private.h" #include "error_private.h"
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ #include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ #define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff_static.h" #include "zbuff.h"
/* ************************************* /* *************************************
@ -95,14 +95,12 @@ struct ZBUFF_CCtx_s {
size_t outBuffContentSize; size_t outBuffContentSize;
size_t outBuffFlushedSize; size_t outBuffFlushedSize;
ZBUFF_cStage stage; ZBUFF_cStage stage;
ZSTD_allocFunction customAlloc; ZSTD_customMem customMem;
ZSTD_freeFunction customFree;
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
ZBUFF_CCtx* ZBUFF_createCCtx(void) ZBUFF_CCtx* ZBUFF_createCCtx(void)
{ {
ZSTD_customMem customMem = { NULL, NULL }; return ZBUFF_createCCtx_advanced(defaultCustomMem);
return ZBUFF_createCCtx_advanced(customMem);
} }
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
@ -110,24 +108,17 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
ZBUFF_CCtx* zbc; ZBUFF_CCtx* zbc;
if (!customMem.customAlloc && !customMem.customFree) if (!customMem.customAlloc && !customMem.customFree)
{ customMem = defaultCustomMem;
zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx));
if (zbc==NULL) return NULL;
zbc->customAlloc = malloc;
zbc->customFree = free;
zbc->zc = ZSTD_createCCtx();
return zbc;
}
if (!customMem.customAlloc || !customMem.customFree) if (!customMem.customAlloc || !customMem.customFree)
return NULL; return NULL;
zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
if (zbc==NULL) return NULL; if (zbc==NULL) return NULL;
memset(zbc, 0, sizeof(ZBUFF_CCtx)); memset(zbc, 0, sizeof(ZBUFF_CCtx));
zbc->customAlloc = customMem.customAlloc; memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
zbc->customFree = customMem.customFree;
zbc->zc = ZSTD_createCCtx_advanced(customMem); zbc->zc = ZSTD_createCCtx_advanced(customMem);
if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
return zbc; return zbc;
} }
@ -135,9 +126,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
{ {
if (zbc==NULL) return 0; /* support free on NULL */ if (zbc==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zbc->zc); ZSTD_freeCCtx(zbc->zc);
zbc->customFree(zbc->inBuff); if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
zbc->customFree(zbc->outBuff); if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
zbc->customFree(zbc); zbc->customMem.customFree(zbc->customMem.opaque, zbc);
return 0; return 0;
} }
@ -152,16 +143,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zbc->inBuffSize < neededInBuffSize) { if (zbc->inBuffSize < neededInBuffSize) {
zbc->inBuffSize = neededInBuffSize; zbc->inBuffSize = neededInBuffSize;
zbc->customFree(zbc->inBuff); /* should not be necessary */ zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation); if (zbc->inBuff == NULL) return ERROR(memory_allocation);
} }
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize);
} }
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
zbc->customFree(zbc->outBuff); /* should not be necessary */ zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation); if (zbc->outBuff == NULL) return ERROR(memory_allocation);
} }
@ -180,9 +171,8 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters params; ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 0;
ZSTD_adjustCParams(&params.cParams, 0, dictSize);
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
} }
@ -192,6 +182,15 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
} }
/* internal util function */
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
/* *** Compression *** */ /* *** Compression *** */
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,

View File

@ -51,12 +51,15 @@
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include <stdlib.h> /* malloc */
#include <string.h> /* memset */ #include <string.h> /* memset */
#include "mem.h" #include "mem.h"
#include "fse_static.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "huf_static.h" #include "xxhash.h" /* XXH_reset, update, digest */
#include "zstd_internal.h" #define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_internal.h" /* includes zstd.h */
/*-************************************* /*-*************************************
@ -70,6 +73,27 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
***************************************/ ***************************************/
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
static U32 ZSTD_highbit32(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-************************************* /*-*************************************
* Sequence storage * Sequence storage
@ -99,12 +123,14 @@ struct ZSTD_CCtx_s
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; U32 loadedDictEnd;
U32 stage; /* 0: created; 1: init,dictLoad; 2:started */ U32 stage; /* 0: created; 1: init,dictLoad; 2:started */
U32 dictID;
ZSTD_parameters params; ZSTD_parameters params;
void* workSpace; void* workSpace;
size_t workSpaceSize; size_t workSpaceSize;
size_t blockSize; size_t blockSize;
ZSTD_allocFunction customAlloc; U64 frameContentSize;
ZSTD_freeFunction customFree; XXH64_state_t xxhState;
ZSTD_customMem customMem;
seqStore_t seqStore; /* sequences storage ptrs */ seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable; U32* hashTable;
@ -119,8 +145,7 @@ struct ZSTD_CCtx_s
ZSTD_CCtx* ZSTD_createCCtx(void) ZSTD_CCtx* ZSTD_createCCtx(void)
{ {
ZSTD_customMem customMem = { NULL, NULL }; return ZSTD_createCCtx_advanced(defaultCustomMem);
return ZSTD_createCCtx_advanced(customMem);
} }
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
@ -128,31 +153,23 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
ZSTD_CCtx* ctx; ZSTD_CCtx* ctx;
if (!customMem.customAlloc && !customMem.customFree) if (!customMem.customAlloc && !customMem.customFree)
{ customMem = defaultCustomMem;
ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
if (!ctx) return NULL;
ctx->customAlloc = malloc;
ctx->customFree = free;
return ctx;
}
if (!customMem.customAlloc || !customMem.customFree) if (!customMem.customAlloc || !customMem.customFree)
return NULL; return NULL;
ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx));
if (!ctx) return NULL; if (!ctx) return NULL;
memset(ctx, 0, sizeof(ZSTD_CCtx)); memset(ctx, 0, sizeof(ZSTD_CCtx));
ctx->customAlloc = customMem.customAlloc; memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem));
ctx->customFree = customMem.customFree;
return ctx; return ctx;
} }
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{ {
cctx->customFree(cctx->workSpace); if (cctx==NULL) return 0; /* support free on NULL */
cctx->customFree(cctx); if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace);
cctx->customMem.customFree(cctx->customMem.opaque, cctx);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
@ -183,8 +200,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
} }
static unsigned ZSTD_highbit(U32 val);
/** ZSTD_checkCParams_advanced() : /** ZSTD_checkCParams_advanced() :
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */ temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize) size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
@ -198,30 +213,32 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
} }
/** ZSTD_adjustParams() : /** ZSTD_adjustCParams() :
optimize params for q given input (`srcSize` and `dictSize`). optimize cPar for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization. mostly downsizing to reduce memory consumption and initialization.
Both `srcSize` and `dictSize` are optional (use 0 if unknown), Both `srcSize` and `dictSize` are optional (use 0 if unknown),
but if both are 0, no optimization can be done. but if both are 0, no optimization can be done.
Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize)
{ {
if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */ if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
/* resize params, to use less memory when necessary */ /* resize params, to use less memory when necessary */
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; { U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1; U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} } } }
if (params->hashLog > params->windowLog) params->hashLog = params->windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt); { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt);
U32 const maxChainLog = params->windowLog+btPlus; U32 const maxChainLog = cPar.windowLog+btPlus;
if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
return cPar;
} }
@ -229,6 +246,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
{ {
ZSTD_CCtx* zc = ZSTD_createCCtx(); ZSTD_CCtx* zc = ZSTD_createCCtx();
ZSTD_parameters params; ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = cParams; params.cParams = cParams;
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0); ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
@ -240,7 +258,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
/*! ZSTD_resetCCtx_advanced() : /*! ZSTD_resetCCtx_advanced() :
note : 'params' is expected to be validated */ note : 'params' is expected to be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, U32 reset) ZSTD_parameters params, U64 frameContentSize, U32 reset)
{ /* note : params considered validated here */ { /* note : params considered validated here */
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
@ -248,7 +266,10 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t tokenSpace = blockSize + 11*maxNbSeq;
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
const size_t hSize = ((size_t)1) << params.cParams.hashLog; const size_t hSize = ((size_t)1) << params.cParams.hashLog;
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 :
( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX :
((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) );
const size_t h3Size = ((size_t)1) << hashLog3;
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
/* Check if workSpace is large enough, alloc a new one if needed */ /* Check if workSpace is large enough, alloc a new one if needed */
@ -257,17 +278,19 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) { if (zc->workSpaceSize < neededSpace) {
zc->customFree(zc->workSpace); zc->customMem.customFree(zc->customMem.opaque, zc->workSpace);
zc->workSpace = zc->customAlloc(neededSpace); zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace);
if (zc->workSpace == NULL) return ERROR(memory_allocation); if (zc->workSpace == NULL) return ERROR(memory_allocation);
zc->workSpaceSize = neededSpace; zc->workSpaceSize = neededSpace;
} } } }
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
zc->hashTable3 = (U32*)(zc->workSpace); XXH64_reset(&zc->xxhState, 0);
zc->hashTable = zc->hashTable3 + h3Size; zc->hashLog3 = hashLog3;
zc->hashTable = (U32*)(zc->workSpace);
zc->chainTable = zc->hashTable + hSize; zc->chainTable = zc->hashTable + hSize;
zc->seqStore.buffer = zc->chainTable + chainSize; zc->hashTable3 = zc->chainTable + chainSize;
zc->seqStore.buffer = zc->hashTable3 + h3Size;
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer; zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
zc->flagStaticTables = 0; zc->flagStaticTables = 0;
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256; zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
@ -280,6 +303,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->lowLimit = 0; zc->lowLimit = 0;
zc->params = params; zc->params = params;
zc->blockSize = blockSize; zc->blockSize = blockSize;
zc->frameContentSize = frameContentSize;
if (params.cParams.strategy == ZSTD_btopt) { if (params.cParams.strategy == ZSTD_btopt) {
zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer); zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer);
@ -300,6 +324,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq; zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
zc->stage = 1; zc->stage = 1;
zc->dictID = 0;
zc->loadedDictEnd = 0; zc->loadedDictEnd = 0;
return 0; return 0;
@ -314,21 +339,19 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
{ {
if (srcCCtx->stage!=1) return ERROR(stage_wrong); if (srcCCtx->stage!=1) return ERROR(stage_wrong);
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
dstCCtx->customAlloc = srcCCtx->customAlloc; ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
dstCCtx->customFree = srcCCtx->customFree;
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
/* copy tables */ /* copy tables */
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; const size_t h3Size = (size_t)1 << srcCCtx->hashLog3;
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
} }
/* copy dictionary pointers */ /* copy dictionary offsets */
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
dstCCtx->nextSrc = srcCCtx->nextSrc; dstCCtx->nextSrc = srcCCtx->nextSrc;
@ -337,6 +360,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
dstCCtx->dictID = srcCCtx->dictID;
/* copy entropy tables */ /* copy entropy tables */
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
@ -384,8 +408,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
/* Frame format description /* Frame format description
Frame Header - [ Block Header - Block ] - Frame End Frame Header - [ Block Header - Block ] - Frame End
1) Frame Header 1) Frame Header
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) - 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
- 1 byte - Frame Descriptor - 1 byte : Frame Header Descriptor
- 1-13 bytes : Optional fields
2) Block Header 2) Block Header
- 3 bytes, starting with a 2-bits descriptor - 3 bytes, starting with a 2-bits descriptor
Uncompressed, Compressed, Frame End, unused Uncompressed, Compressed, Frame End, unused
@ -398,13 +423,38 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
/* Frame descriptor /* Frame descriptor
1 byte, using : // old
1 byte - Alloc :
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
bit 4 : minmatch 4(0) or 3(1) bit 4 : reserved for windowLog (must be zero)
bit 5 : reserved (must be zero) bit 5 : reserved (must be zero)
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
Optional : content size (0, 1, 2 or 8 bytes) 1 byte - checker :
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
bit 2-7 : reserved (must be zero)
// new
1 byte - FrameHeaderDescription :
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
bit 2-4 : reserved (must be zero)
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8)
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
Optional : WindowLog (0 or 1 byte)
bit 0-2 : octal Fractional (1/8th)
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
Optional : dictID (0, 1, 2 or 4 bytes)
Automatic adaptation
0 : no dictID
1 : 1 - 255
2 : 256 - 65535
4 : all other values
Optional : content size (0, 1, 2, 4 or 8 bytes)
0 : unknown 0 : unknown
1 : 0-255 bytes 1 : 0-255 bytes
2 : 256 - 65535+256 2 : 256 - 65535+256
@ -651,7 +701,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
size_t u; size_t u;
for (u=0; u<nbSeq; u++) { for (u=0; u<nbSeq; u++) {
U32 const ll = llTable[u]; U32 const ll = llTable[u];
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll]; llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll];
} }
if (seqStorePtr->longLengthID==1) if (seqStorePtr->longLengthID==1)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL; llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
@ -661,7 +711,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
{ const U32* const offsetTable = seqStorePtr->offsetStart; { const U32* const offsetTable = seqStorePtr->offsetStart;
BYTE* const ofCodeTable = seqStorePtr->offCodeStart; BYTE* const ofCodeTable = seqStorePtr->offCodeStart;
size_t u; size_t u;
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit(offsetTable[u]); for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit32(offsetTable[u]);
} }
/* ML codes */ /* ML codes */
@ -679,7 +729,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
size_t u; size_t u;
for (u=0; u<nbSeq; u++) { for (u=0; u<nbSeq; u++) {
U32 const ml = mlTable[u]; U32 const ml = mlTable[u];
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml]; mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml];
} }
if (seqStorePtr->longLengthID==2) if (seqStorePtr->longLengthID==2)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML; mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
@ -1427,7 +1477,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
if (matchLength > bestLength) { if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) ) if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
@ -1713,14 +1763,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
int const gain2 = (int)(mlRep * 3); int const gain2 = (int)(mlRep * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
matchLength = mlRep, offset = 0, start = ip; matchLength = mlRep, offset = 0, start = ip;
} }
{ size_t offset2=99999999; { size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
@ -1732,14 +1782,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32; size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
int const gain2 = (int)(ml2 * 4); int const gain2 = (int)(ml2 * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
matchLength = ml2, offset = 0, start = ip; matchLength = ml2, offset = 0, start = ip;
} }
{ size_t offset2=99999999; { size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
@ -1885,7 +1935,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
int const gain2 = (int)(repLength * 3); int const gain2 = (int)(repLength * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } } }
@ -1893,8 +1943,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 1 */ /* search match, depth 1 */
{ size_t offset2=99999999; { size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */ continue; /* search a better one */
@ -1915,7 +1965,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
int gain2 = (int)(repLength * 4); int gain2 = (int)(repLength * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip; matchLength = repLength, offset = 0, start = ip;
} } } }
@ -1923,8 +1973,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 2 */ /* search match, depth 2 */
{ size_t offset2=99999999; { size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */ int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip; matchLength = ml2, offset = offset2, start = ip;
continue; continue;
@ -1934,7 +1984,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* catch up */ /* catch up */
if (offset) { if (offset) {
U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
@ -2039,34 +2089,37 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
size_t blockSize = zc->blockSize; size_t blockSize = cctx->blockSize;
size_t remaining = srcSize; size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart; BYTE* op = ostart;
const U32 maxDist = 1 << zc->params.cParams.windowLog; const U32 maxDist = 1 << cctx->params.cParams.windowLog;
ZSTD_stats_t* stats = &zc->seqStore.stats; ZSTD_stats_t* stats = &cctx->seqStore.stats;
ZSTD_statsInit(stats); ZSTD_statsInit(stats);
if (cctx->params.fParams.checksumFlag)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) { while (remaining) {
size_t cSize; size_t cSize;
ZSTD_statsResetFreqs(stats); ZSTD_statsResetFreqs(stats); /* debug only */
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining; if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */ /* enforce maxDist */
U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist; U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit; if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit; if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
} }
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */ if (cSize == 0) { /* block is not compressible */
@ -2086,33 +2139,47 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
op += cSize; op += cSize;
} }
ZSTD_statsPrint(stats, zc->params.cParams.searchLength); ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
return op-ostart; return op-ostart;
} }
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
ZSTD_parameters params, U64 pledgedSrcSize) ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
{ BYTE* const op = (BYTE*)dst; { BYTE* const op = (BYTE*)dst;
U32 const fcsId = params.fParams.contentSizeFlag ? U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
(pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */ U32 const checksumFlag = params.fParams.checksumFlag;
U32 const windowSize = 1U << params.cParams.windowLog;
U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
U32 const fcsCode = params.fParams.contentSizeFlag ?
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
0; 0;
BYTE const fdescriptor = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) );
| (fcsId << 6) ); size_t pos;
size_t const hSize = ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
if (hSize > dstCapacity) return ERROR(dstSize_tooSmall); if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
MEM_writeLE32(dst, ZSTD_MAGICNUMBER); MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
op[4] = fdescriptor; op[4] = frameHeaderDecriptionByte; pos=5;
switch(fcsId) if (!directModeFlag) op[pos++] = windowLogByte;
switch(dictIDSizeCode)
{ {
default: /* impossible */ default: /* impossible */
case 0 : break; case 0 : break;
case 1 : op[5] = (BYTE)(pledgedSrcSize); break; case 1 : op[pos] = (BYTE)(dictID); pos++; break;
case 2 : MEM_writeLE16(op+5, (U16)(pledgedSrcSize-256)); break; case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break;
case 3 : MEM_writeLE64(op+5, (U64)(pledgedSrcSize)); break; case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
} }
return hSize; switch(fcsCode)
{
default: /* impossible */
case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break;
case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
}
return pos;
} }
@ -2126,7 +2193,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
if (zc->stage==0) return ERROR(stage_wrong); if (zc->stage==0) return ERROR(stage_wrong);
if (frame && (zc->stage==1)) { /* copy saved header */ if (frame && (zc->stage==1)) { /* copy saved header */
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
dst = (char*)dst + fhSize; dst = (char*)dst + fhSize;
@ -2244,53 +2311,59 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{ {
/* note : magic number already checked */ /* note : magic number already checked */
size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode; size_t const dictSizeStart = dictSize;
short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize); { size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
zc->flagStaticTables = 1; zc->flagStaticTables = 1;
dict = (const char*)dict + hufHeaderSize; dict = (const char*)dict + hufHeaderSize;
dictSize -= hufHeaderSize; dictSize -= hufHeaderSize;
}
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); { short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); { size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize; dict = (const char*)dict + offcodeHeaderSize;
dictSize -= offcodeHeaderSize; dictSize -= offcodeHeaderSize;
}
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); { size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize; dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize; dictSize -= matchlengthHeaderSize;
}
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); { size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictSize -= litlengthHeaderSize;
}
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; return (dictSizeStart-dictSize);
} }
/** ZSTD_compress_insertDictionary() : /** ZSTD_compress_insertDictionary() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{ {
if ((dict==NULL) || (dictSize<=4)) return 0; if ((dict==NULL) || (dictSize<=8)) return 0;
/* default : dict is pure content */ /* default : dict is pure content */
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
/* known magic number : dict is parsed for entropy stats and content */ /* known magic number : dict is parsed for entropy stats and content */
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4; { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8;
if (ZSTD_isError(eSize)) return eSize; if (ZSTD_isError(eSize)) return eSize;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
} }
@ -2303,11 +2376,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize) ZSTD_parameters params, U64 pledgedSrcSize)
{ {
{ U32 const hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; } if (ZSTD_isError(resetError)) return resetError;
{ size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1);
if (ZSTD_isError(resetError)) return resetError; }
return ZSTD_compress_insertDictionary(zc, dict, dictSize); return ZSTD_compress_insertDictionary(zc, dict, dictSize);
} }
@ -2330,9 +2400,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters params; ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 0;
ZSTD_adjustCParams(&params.cParams, 0, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
} }
@ -2348,30 +2417,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
/*! ZSTD_compressEnd() : /*! ZSTD_compressEnd() :
* Write frame epilogue. * Write frame epilogue.
* @return : nb of bytes written into dst (or an error code) */ * @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
{ {
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
size_t fhSize = 0; size_t fhSize = 0;
/* not even init ! */ /* not even init ! */
if (zc->stage==0) return ERROR(stage_wrong); if (cctx->stage==0) return ERROR(stage_wrong);
/* special case : empty frame */ /* special case : empty frame */
if (zc->stage==1) { if (cctx->stage==1) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0); fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
op += fhSize; op += fhSize;
zc->stage = 2; cctx->stage = 2;
} }
/* frame epilogue */ /* frame epilogue */
if (dstCapacity < 3) return ERROR(dstSize_tooSmall); if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
op[0] = (BYTE)(bt_end << 6); { U32 const checksum = cctx->params.fParams.checksumFlag ?
op[1] = 0; (U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
op[2] = 0; 0;
op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
op[1] = (BYTE)(checksum>>8);
op[2] = (BYTE)checksum;
}
zc->stage = 0; /* return to "created by not init" status */ cctx->stage = 0; /* return to "created but not init" status */
return 3+fhSize; return 3+fhSize;
} }
@ -2434,10 +2507,10 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters params; ZSTD_parameters params;
memset(&params, 0, sizeof(params));
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
ZSTD_adjustCParams(&params.cParams, srcSize, dictSize);
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
} }
@ -2452,10 +2525,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
size_t result; size_t result;
ZSTD_CCtx ctxBody; ZSTD_CCtx ctxBody;
memset(&ctxBody, 0, sizeof(ctxBody)); memset(&ctxBody, 0, sizeof(ctxBody));
ctxBody.customAlloc = malloc; memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
ctxBody.customFree = free;
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
return result; return result;
} }
@ -2590,5 +2662,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
} }
cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
return cp; return cp;
} }

View File

@ -41,10 +41,10 @@
***************************************/ ***************************************/
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
{ {
ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1); ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1);
ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1); ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1);
ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1); ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1);
ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1); ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1);
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
} }
@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
U32 price, u; U32 price, u;
if (litLength == 0) if (litLength == 0)
return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
/* literals */ /* literals */
if (ssPtr->cachedLiterals == literals) { if (ssPtr->cachedLiterals == literals) {
@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
for (u=0; u < additional; u++) for (u=0; u < additional; u++)
price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1);
ssPtr->cachedPrice = price; ssPtr->cachedPrice = price;
ssPtr->cachedLitLength = litLength; ssPtr->cachedLitLength = litLength;
} else { } else {
price = litLength * ssPtr->log2litSum; price = litLength * ssPtr->log2litSum;
for (u=0; u < litLength; u++) for (u=0; u < litLength; u++)
price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1);
if (litLength >= 12) { if (litLength >= 12) {
ssPtr->cachedLiterals = literals; ssPtr->cachedLiterals = literals;
@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 }; 24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19; const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1);
} }
return price; return price;
@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{ {
/* offset */ /* offset */
BYTE offCode = (BYTE)ZSTD_highbit(offset+1); BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
/* match Length */ /* match Length */
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36; const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1);
} }
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 }; 24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19; const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++; seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++; seqStorePtr->litLengthSum++;
} }
/* match offset */ /* match offset */
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1); { BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
seqStorePtr->offCodeSum++; seqStorePtr->offCodeSum++;
seqStorePtr->offCodeFreq[offCode]++; seqStorePtr->offCodeFreq[offCode]++;
} }
@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36; const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++; seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++; seqStorePtr->matchLengthSum++;
} }

View File

@ -59,12 +59,12 @@
/* ************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */ #include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */ #include <stdio.h> /* printf (debug) */
#include "huf_static.h"
#include "bitstream.h" #include "bitstream.h"
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"

View File

@ -35,9 +35,9 @@
***************************************/ ***************************************/
#include <stdlib.h> #include <stdlib.h>
#include "error_private.h" #include "error_private.h"
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ #include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ #define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff_static.h" #include "zbuff.h"
/*-*************************************************************************** /*-***************************************************************************
@ -82,15 +82,13 @@ struct ZBUFF_DCtx_s {
size_t blockSize; size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize; size_t lhSize;
ZSTD_allocFunction customAlloc; ZSTD_customMem customMem;
ZSTD_freeFunction customFree;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
ZBUFF_DCtx* ZBUFF_createDCtx(void) ZBUFF_DCtx* ZBUFF_createDCtx(void)
{ {
ZSTD_customMem customMem = { NULL, NULL }; return ZBUFF_createDCtx_advanced(defaultCustomMem);
return ZBUFF_createDCtx_advanced(customMem);
} }
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
@ -98,25 +96,17 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
ZBUFF_DCtx* zbd; ZBUFF_DCtx* zbd;
if (!customMem.customAlloc && !customMem.customFree) if (!customMem.customAlloc && !customMem.customFree)
{ customMem = defaultCustomMem;
zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL;
zbd->customAlloc = malloc;
zbd->customFree = free;
zbd->zd = ZSTD_createDCtx();
zbd->stage = ZBUFFds_init;
return zbd;
}
if (!customMem.customAlloc || !customMem.customFree) if (!customMem.customAlloc || !customMem.customFree)
return NULL; return NULL;
zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL; if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(ZBUFF_DCtx)); memset(zbd, 0, sizeof(ZBUFF_DCtx));
zbd->customAlloc = customMem.customAlloc; memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
zbd->customFree = customMem.customFree;
zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->zd = ZSTD_createDCtx_advanced(customMem);
if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
zbd->stage = ZBUFFds_init; zbd->stage = ZBUFFds_init;
return zbd; return zbd;
} }
@ -125,9 +115,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
{ {
if (zbd==NULL) return 0; /* support free on null */ if (zbd==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbd->zd); ZSTD_freeDCtx(zbd->zd);
zbd->customFree(zbd->inBuff); if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
zbd->customFree(zbd->outBuff); if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->customFree(zbd); zbd->customMem.customFree(zbd->customMem.opaque, zbd);
return 0; return 0;
} }
@ -147,6 +137,15 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
} }
/* internal util function */
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
/* *** Decompression *** */ /* *** Decompression *** */
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
@ -192,20 +191,22 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
if (ZSTD_isError(h2Result)) return h2Result; if (ZSTD_isError(h2Result)) return h2Result;
} } } }
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
/* Frame header instruct buffer sizes */ /* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
zbd->blockSize = blockSize; zbd->blockSize = blockSize;
if (zbd->inBuffSize < blockSize) { if (zbd->inBuffSize < blockSize) {
zbd->customFree(zbd->inBuff); zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
zbd->inBuffSize = blockSize; zbd->inBuffSize = blockSize;
zbd->inBuff = (char*)zbd->customAlloc(blockSize); zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
if (zbd->inBuff == NULL) return ERROR(memory_allocation); if (zbd->inBuff == NULL) return ERROR(memory_allocation);
} }
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; { size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
if (zbd->outBuffSize < neededOutSize) { if (zbd->outBuffSize < neededOutSize) {
zbd->customFree(zbd->outBuff); zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->outBuffSize = neededOutSize; zbd->outBuffSize = neededOutSize;
zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
if (zbd->outBuff == NULL) return ERROR(memory_allocation); if (zbd->outBuff == NULL) return ERROR(memory_allocation);
} } } } } }
zbd->stage = ZBUFFds_read; zbd->stage = ZBUFFds_read;
@ -218,12 +219,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
break; break;
} }
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
ip, neededInSize); ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize; ip += neededInSize;
if (!decodedSize) break; /* this was just a header */ if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize; zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush; zbd->stage = ZBUFFds_flush;
break; break;
@ -243,12 +245,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */ /* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->inBuff, neededInSize); zbd->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
zbd->inPos = 0; /* input is consumed */ zbd->inPos = 0; /* input is consumed */
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize; zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush; zbd->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */ // break; /* ZBUFFds_flush follows */
@ -276,7 +279,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
*srcSizePtr = ip-istart; *srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart; *dstCapacityPtr = op-ostart;
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ // if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
nextSrcSizeHint -= zbd->inPos; /* already loaded*/ nextSrcSizeHint -= zbd->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;
} }

View File

@ -53,15 +53,18 @@
/*-******************************************************* /*-*******************************************************
* Dependencies * Dependencies
*********************************************************/ *********************************************************/
#include <stdlib.h> /* calloc */
#include <string.h> /* memcpy, memmove */ #include <string.h> /* memcpy, memmove */
#include <stdio.h> /* debug only : printf */ #include <stdio.h> /* debug only : printf */
#include "mem.h" /* low level memory routines */ #include "mem.h" /* low level memory routines */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_internal.h" #include "zstd_internal.h"
#include "fse_static.h"
#include "huf_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
# include "zstd_legacy.h" # include "zstd_legacy.h"
#endif #endif
@ -101,7 +104,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
* Context management * Context management
***************************************************************/ ***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s
{ {
@ -116,10 +120,11 @@ struct ZSTD_DCtx_s
size_t expected; size_t expected;
size_t headerSize; size_t headerSize;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
ZSTD_allocFunction customAlloc; XXH64_state_t xxhState;
ZSTD_freeFunction customFree; ZSTD_customMem customMem;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage; ZSTD_dStage stage;
U32 dictID;
U32 flagRepeatTable; U32 flagRepeatTable;
const BYTE* litPtr; const BYTE* litPtr;
size_t litBufSize; size_t litBufSize;
@ -140,46 +145,37 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->dictEnd = NULL; dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog; dctx->hufTableX4[0] = HufLog;
dctx->flagRepeatTable = 0; dctx->flagRepeatTable = 0;
dctx->dictID = 0;
return 0; return 0;
} }
ZSTD_DCtx* ZSTD_createDCtx(void)
{
ZSTD_customMem customMem = { NULL, NULL };
return ZSTD_createDCtx_advanced(customMem);
}
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{ {
ZSTD_DCtx* dctx; ZSTD_DCtx* dctx;
if (!customMem.customAlloc && !customMem.customFree) if (!customMem.customAlloc && !customMem.customFree)
{ customMem = defaultCustomMem;
dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
dctx->customAlloc = malloc;
dctx->customFree = free;
ZSTD_decompressBegin(dctx);
return dctx;
}
if (!customMem.customAlloc || !customMem.customFree) if (!customMem.customAlloc || !customMem.customFree)
return NULL; return NULL;
dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
if (!dctx) return NULL; if (!dctx) return NULL;
dctx->customAlloc = customMem.customAlloc; memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
dctx->customFree = customMem.customFree;
ZSTD_decompressBegin(dctx); ZSTD_decompressBegin(dctx);
return dctx; return dctx;
} }
ZSTD_DCtx* ZSTD_createDCtx(void)
{
return ZSTD_createDCtx_advanced(defaultCustomMem);
}
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
{ {
dctx->customFree(dctx); if (dctx==NULL) return 0; /* support free on NULL */
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
return 0; /* reserved as a potential error code in the future */ return 0; /* reserved as a potential error code in the future */
} }
@ -211,17 +207,42 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
/* Frame descriptor /* Frame descriptor
1 byte, using : // old
1 byte - Alloc :
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
bit 4 : minmatch 4(0) or 3(1) bit 4 : reserved for windowLog (must be zero)
bit 5 : reserved (must be zero) bit 5 : reserved (must be zero)
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
Optional : content size (0, 1, 2 or 8 bytes) 1 byte - checker :
0 : unknown bit 0-1 : dictID (0, 1, 2 or 4 bytes)
1 : 0-255 bytes bit 2-7 : reserved (must be zero)
2 : 256 - 65535+256
8 : up to 16 exa // new
1 byte - FrameHeaderDescription :
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
bit 2-4 : reserved (must be zero)
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
Optional : WindowLog (0 or 1 byte)
bit 0-2 : octal Fractional (1/8th)
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
Optional : dictID (0, 1, 2 or 4 bytes)
Automatic adaptation
0 : no dictID
1 : 1 - 255
2 : 256 - 65535
4 : all other values
Optional : content size (0, 1, 2, 4 or 8 bytes)
0 : unknown (fcfs==0 and swl==0)
1 : 0-255 bytes (fcfs==0 and swl==1)
2 : 256 - 65535+256 (fcfs==1)
4 : 0 - 4GB-1 (fcfs==2)
8 : 0 - 16EB-1 (fcfs==3)
*/ */
@ -297,13 +318,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{ {
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6; { BYTE const fhd = ((const BYTE*)src)[4];
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } U32 const dictID= fhd & 3;
U32 const directMode = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6;
return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ (directMode && !ZSTD_fcs_fieldSize[fcsId]);
}
} }
/** ZSTD_getFrameParams() : /** ZSTD_getFrameParams() :
* decode Frame Header, or provide expected `srcSize`. * decode Frame Header, or require larger `srcSize`.
* @return : 0, `fparamsPtr` is correctly filled, * @return : 0, `fparamsPtr` is correctly filled,
* >0, `srcSize` is too small, result is expected `srcSize`, * >0, `srcSize` is too small, result is expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
@ -312,24 +338,63 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
return 0;
}
return ERROR(prefix_unknown);
}
/* ensure there is enough `srcSize` to fully read/decode frame header */ /* ensure there is enough `srcSize` to fully read/decode frame header */
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
if (srcSize < fhsize) return fhsize; } if (srcSize < fhsize) return fhsize; }
memset(fparamsPtr, 0, sizeof(*fparamsPtr)); { BYTE const fhdByte = ip[4];
{ BYTE const frameDesc = ip[4]; size_t pos = 5;
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; U32 const dictIDSizeCode = fhdByte&3;
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ U32 const checksumFlag = (fhdByte>>2)&1;
switch(frameDesc >> 6) /* fcsId */ U32 const directMode = (fhdByte>>5)&1;
U32 const fcsID = fhdByte>>6;
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
U32 windowSize = 0;
U32 dictID = 0;
U64 frameContentSize = 0;
if ((fhdByte & 0x18) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
if (!directMode) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
switch(dictIDSizeCode)
{ {
default: /* impossible */ default: /* impossible */
case 0 : fparamsPtr->frameContentSize = 0; break; case 0 : break;
case 1 : fparamsPtr->frameContentSize = ip[5]; break; case 1 : dictID = ip[pos]; pos++; break;
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
} } }
switch(fcsID)
{
default: /* impossible */
case 0 : if (directMode) frameContentSize = ip[pos]; break;
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
fparamsPtr->checksumFlag = checksumFlag;
}
return 0; return 0;
} }
@ -337,10 +402,11 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
/** ZSTD_decodeFrameHeader() : /** ZSTD_decodeFrameHeader() :
* `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * `srcSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
{ {
size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
return result; return result;
} }
@ -611,9 +677,10 @@ typedef struct {
static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) static seq_t ZSTD_decodeSequence(seqState_t* seqState)
{ {
/* Literal length */ seq_t seq;
U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
U32 const mlCode = FSE_peekSymbol(&(seqState->stateML)); U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
@ -669,13 +736,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset; seqState->prevOffset[0] = offset;
} }
seq->offset = offset; seq.offset = offset;
} }
seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
if (MEM_32bits() || if (MEM_32bits() ||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
@ -684,6 +751,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
return seq;
} }
@ -789,11 +858,7 @@ static size_t ZSTD_decompressSequences(
/* Regen sequences */ /* Regen sequences */
if (nbSeq) { if (nbSeq) {
seq_t sequence;
seqState_t seqState; seqState_t seqState;
memset(&sequence, 0, sizeof(sequence));
sequence.offset = REPCODE_STARTVALUE;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; } { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected); } if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
@ -803,7 +868,7 @@ static size_t ZSTD_decompressSequences(
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--; nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState); { seq_t const sequence = ZSTD_decodeSequence(&seqState);
#if 0 /* debug */ #if 0 /* debug */
static BYTE* start = NULL; static BYTE* start = NULL;
@ -814,7 +879,7 @@ static size_t ZSTD_decompressSequences(
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset); pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
#endif #endif
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } } }
@ -882,10 +947,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
const BYTE* const iend = ip + srcSize; const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstCapacity; BYTE* const oend = ostart + dstCapacity;
BYTE* op = ostart;
size_t remainingSize = srcSize; size_t remainingSize = srcSize;
blockProperties_t blockProperties = { bt_compressed, 0 };
/* check */ /* check */
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
@ -901,6 +965,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
/* Loop on each block */ /* Loop on each block */
while (1) { while (1) {
size_t decodedSize=0; size_t decodedSize=0;
blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize)) return cBlockSize;
@ -929,6 +994,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (cBlockSize == 0) break; /* bt_end */ if (cBlockSize == 0) break; /* bt_end */
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize; op += decodedSize;
ip += cBlockSize; ip += cBlockSize;
remainingSize -= cBlockSize; remainingSize -= cBlockSize;
@ -954,7 +1020,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict, size_t dictSize) const void* dict, size_t dictSize)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
{ const U32 magicNumber = MEM_readLE32(src); { U32 const magicNumber = MEM_readLE32(src);
if (ZSTD_isLegacy(magicNumber)) if (ZSTD_isLegacy(magicNumber))
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber); return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
} }
@ -975,7 +1041,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
{ {
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
size_t regenSize; size_t regenSize;
ZSTD_DCtx* dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
if (dctx==NULL) return ERROR(memory_allocation); if (dctx==NULL) return ERROR(memory_allocation);
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
@ -995,17 +1061,30 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
return dctx->expected; return dctx->expected;
} }
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
{
return dctx->stage == ZSTDds_skipFrame;
}
/** ZSTD_decompressContinue() :
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
/* Sanity check */ /* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
if (dstCapacity) ZSTD_checkContinuity(dctx, dst); if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
/* Decompress : frame header; part 1 */
switch (dctx->stage) switch (dctx->stage)
{ {
case ZSTDds_getFrameHeaderSize : case ZSTDds_getFrameHeaderSize :
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
dctx->stage = ZSTDds_decodeSkippableHeader;
return 0;
}
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
@ -1030,6 +1109,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) { if (bp.blockType == bt_end) {
if (dctx->fParams.checksumFlag) {
U64 const h64 = XXH64_digest(&dctx->xxhState);
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
const BYTE* const ip = (const BYTE*)src;
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
if (check32 != h32) return ERROR(checksum_wrong);
}
dctx->expected = 0; dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
} else { } else {
@ -1061,8 +1147,21 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->stage = ZSTDds_decodeBlockHeader; dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize; dctx->expected = ZSTD_blockHeaderSize;
dctx->previousDstEnd = (char*)dst + rSize; dctx->previousDstEnd = (char*)dst + rSize;
if (ZSTD_isError(rSize)) return rSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
return rSize; return rSize;
} }
case ZSTDds_decodeSkippableHeader:
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame;
return 0;
}
case ZSTDds_skipFrame:
{ dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize;
return 0;
}
default: default:
return ERROR(GENERIC); /* impossible */ return ERROR(GENERIC); /* impossible */
} }
@ -1077,18 +1176,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
dctx->previousDstEnd = (const char*)dict + dictSize; dctx->previousDstEnd = (const char*)dict + dictSize;
} }
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart)
{ {
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize; size_t dictSize = dictSizeStart;
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); { size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + hSize; dict = (const char*)dict + hSize;
dictSize -= hSize; dictSize -= hSize;
}
{ short offcodeNCount[MaxOff+1]; { short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@ -1098,7 +1198,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
{ short matchlengthNCount[MaxML+1]; { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@ -1108,37 +1208,42 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
{ short litlengthNCount[MaxLL+1]; { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictSize -= litlengthHeaderSize;
} }
dctx->flagRepeatTable = 1; dctx->flagRepeatTable = 1;
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; return dictSizeStart - dictSize;
} }
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{ {
size_t eSize; if (dictSize < 8) return ERROR(dictionary_corrupted);
U32 const magic = MEM_readLE32(dict); { U32 const magic = MEM_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) { if (magic != ZSTD_DICT_MAGIC) {
/* pure content mode */ /* pure content mode */
ZSTD_refDictContent(dctx, dict, dictSize); ZSTD_refDictContent(dctx, dict, dictSize);
return 0; return 0;
} }
/* load entropy tables */ dctx->dictID = MEM_readLE32((const char*)dict + 4);
dict = (const char*)dict + 4;
dictSize -= 4;
eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
/* reference dictionary content */ /* load entropy tables */
dict = (const char*)dict + 8;
dictSize -= 8;
{ size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + eSize; dict = (const char*)dict + eSize;
dictSize -= eSize; dictSize -= eSize;
}
/* reference dictionary content */
ZSTD_refDictContent(dctx, dict, dictSize); ZSTD_refDictContent(dctx, dict, dictSize);
return 0; return 0;
}
} }
@ -1154,4 +1259,3 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
return 0; return 0;
} }

View File

@ -59,11 +59,13 @@
#include "mem.h" /* read */ #include "mem.h" /* read */
#include "error_private.h" #include "error_private.h"
#include "fse.h" #include "fse.h"
#include "huf_static.h" #define HUF_STATIC_LINKING_ONLY
#include "zstd_internal.h" #include "huf.h"
#include "zstd_internal.h" /* includes zstd.h */
#include "xxhash.h"
#include "divsufsort.h" #include "divsufsort.h"
#include "zdict_static.h" #define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h"
/*-************************************* /*-*************************************
@ -104,8 +106,7 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
{ {
const BYTE* const b = (const BYTE*)ptr; const BYTE* const b = (const BYTE*)ptr;
size_t u; size_t u;
for (u=0; u<length; u++) for (u=0; u<length; u++) {
{
BYTE c = b[u]; BYTE c = b[u];
if (c<32 || c>126) c = '.'; /* non-printable char */ if (c<32 || c>126) c = '.'; /* non-printable char */
DISPLAYLEVEL(dlevel, "%c", c); DISPLAYLEVEL(dlevel, "%c", c);
@ -198,8 +199,12 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch)
{ {
const char* const pStart = (const char*)pIn; const char* const pStart = (const char*)pIn;
for (;;) { for (;;) {
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn); size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) { pIn = (const char*)pIn+sizeof(size_t); pMatch = (const char*)pMatch+sizeof(size_t); continue; } if (!diff) {
pIn = (const char*)pIn+sizeof(size_t);
pMatch = (const char*)pMatch+sizeof(size_t);
continue;
}
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
return (size_t)((const char*)pIn - pStart); return (size_t)((const char*)pIn - pStart);
} }
@ -346,9 +351,8 @@ static dictItem ZDICT_analyzePos(
maxLength = i; maxLength = i;
/* reduce maxLength in case of final into repetitive data */ /* reduce maxLength in case of final into repetitive data */
{ { U32 l = (U32)maxLength;
U32 l = (U32)maxLength; BYTE const c = b[pos + maxLength-1];
BYTE c = b[pos + maxLength-1];
while (b[pos+l-2]==c) l--; while (b[pos+l-2]==c) l--;
maxLength = l; maxLength = l;
} }
@ -367,12 +371,10 @@ static dictItem ZDICT_analyzePos(
solution.savings = savings[maxLength]; solution.savings = savings[maxLength];
/* mark positions done */ /* mark positions done */
{ { U32 id;
U32 id;
U32 testedPos;
for (id=start; id<end; id++) { for (id=start; id<end; id++) {
U32 p, pEnd; U32 p, pEnd;
testedPos = suffix[id]; U32 const testedPos = suffix[id];
if (testedPos == pos) if (testedPos == pos)
length = solution.length; length = solution.length;
else { else {
@ -439,7 +441,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
static void ZDICT_removeDictItem(dictItem* table, U32 id) static void ZDICT_removeDictItem(dictItem* table, U32 id)
{ {
/* convention : first element is nb of elts */ /* convention : first element is nb of elts */
U32 max = table->pos; U32 const max = table->pos;
U32 u; U32 u;
if (!id) return; /* protection, should never happen */ if (!id) return; /* protection, should never happen */
for (u=id; u<max-1; u++) for (u=id; u<max-1; u++)
@ -463,8 +465,7 @@ static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt)
} }
/* insert */ /* insert */
{ { U32 current;
U32 current;
U32 nextElt = table->pos; U32 nextElt = table->pos;
if (nextElt >= maxSize) nextElt = maxSize-1; if (nextElt >= maxSize) nextElt = maxSize-1;
current = nextElt-1; current = nextElt-1;
@ -530,8 +531,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
DISPLAYLEVEL(2, "finding patterns ... \n"); DISPLAYLEVEL(2, "finding patterns ... \n");
DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio); DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio);
{ { U32 cursor; for (cursor=0; cursor < bufferSize; ) {
U32 cursor; for (cursor=0; cursor < bufferSize; ) {
dictItem solution; dictItem solution;
if (doneMarks[cursor]) { cursor++; continue; } if (doneMarks[cursor]) { cursor++; continue; }
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio); solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
@ -542,8 +542,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
} } } }
/* limit dictionary size */ /* limit dictionary size */
{ { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */
U32 max = dictList->pos; /* convention : nb of useful elts within dictList */
U32 currentSize = 0; U32 currentSize = 0;
U32 n; for (n=1; n<max; n++) { U32 n; for (n=1; n<max; n++) {
currentSize += dictList[n].length; currentSize += dictList[n].length;
@ -785,7 +784,7 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
{ {
char* dstPtr = (char*)dictBuffer + dictSize; char* dstPtr = (char*)dictBuffer + dictSize;
const char* srcPtr = (const char*)samplesBuffer; const char* srcPtr = (const char*)samplesBuffer;
size_t nbSegments = dictSize / DIB_FASTSEGMENTSIZE; size_t const nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
size_t segNb, interSize; size_t segNb, interSize;
if (nbSegments <= 2) return ERROR(srcSize_wrong); if (nbSegments <= 2) return ERROR(srcSize_wrong);
@ -810,6 +809,33 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
return nbSegments * DIB_FASTSEGMENTSIZE; return nbSegments * DIB_FASTSEGMENTSIZE;
} }
size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params)
{
size_t hSize;
unsigned const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
/* dictionary header */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11);
MEM_writeLE32((char*)dictBuffer+4, dictID);
}
hSize = 8;
/* entropy tables */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
DISPLAYLEVEL(2, "statistics ... \n");
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
compressionLevel,
samplesBuffer, samplesSizes, nbSamples,
(char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
if (hSize + dictContentSize < dictBufferCapacity)
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
return MIN(dictBufferCapacity, hSize+dictContentSize);
}
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3) #define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
/*! ZDICT_trainFromBuffer_unsafe() : /*! ZDICT_trainFromBuffer_unsafe() :
@ -818,14 +844,13 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
*/ */
size_t ZDICT_trainFromBuffer_unsafe( size_t ZDICT_trainFromBuffer_unsafe(
void* dictBuffer, size_t maxDictSize, void* dictBuffer, size_t maxDictSize,
const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params) ZDICT_params_t params)
{ {
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16)); U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
unsigned selectivity = params.selectivityLevel; unsigned selectivity = params.selectivityLevel;
unsigned compressionLevel = params.compressionLevel; size_t const targetDictSize = maxDictSize;
size_t targetDictSize = maxDictSize;
size_t sBuffSize; size_t sBuffSize;
size_t dictSize = 0; size_t dictSize = 0;
@ -834,18 +859,17 @@ size_t ZDICT_trainFromBuffer_unsafe(
if (!dictList) return ERROR(memory_allocation); if (!dictList) return ERROR(memory_allocation);
/* init */ /* init */
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; } { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */ if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
ZDICT_initDictItem(dictList); ZDICT_initDictItem(dictList);
g_displayLevel = params.notificationLevel; g_displayLevel = params.notificationLevel;
if (selectivity==0) selectivity = g_selectivity_default; if (selectivity==0) selectivity = g_selectivity_default;
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
/* build dictionary */ /* build dictionary */
if (selectivity>1) { /* selectivity == 1 => fast mode */ if (selectivity>1) { /* selectivity == 1 => fast mode */
ZDICT_trainBuffer(dictList, dictListSize, ZDICT_trainBuffer(dictList, dictListSize,
samplesBuffer, sBuffSize, samplesBuffer, sBuffSize,
sampleSizes, nbSamples, samplesSizes, nbSamples,
selectivity, (U32)targetDictSize); selectivity, (U32)targetDictSize);
/* display best matches */ /* display best matches */
@ -867,18 +891,16 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* create dictionary */ /* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList); { U32 dictContentSize = ZDICT_dictSize(dictList);
size_t hSize;
BYTE* ptr;
U32 u;
/* build dict content */ /* build dict content */
ptr = (BYTE*)dictBuffer + maxDictSize; { U32 u;
BYTE* ptr = (BYTE*)dictBuffer + maxDictSize;
for (u=1; u<dictList->pos; u++) { for (u=1; u<dictList->pos; u++) {
U32 l = dictList[u].length; U32 l = dictList[u].length;
ptr -= l; ptr -= l;
if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
} } }
/* fast mode dict content */ /* fast mode dict content */
if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */ if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */
@ -888,21 +910,9 @@ size_t ZDICT_trainFromBuffer_unsafe(
samplesBuffer, sBuffSize); samplesBuffer, sBuffSize);
} }
/* dictionary header */ dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize,
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); samplesBuffer, samplesSizes, nbSamples,
hSize = 4; params);
/* entropic tables */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
DISPLAYLEVEL(2, "statistics ... \n");
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+4, maxDictSize-4,
compressionLevel,
samplesBuffer, sampleSizes, nbSamples,
(char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
if (hSize + dictContentSize < maxDictSize)
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
dictSize = MIN(maxDictSize, hSize+dictContentSize);
} }
/* clean up */ /* clean up */
@ -947,3 +957,12 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
params); params);
} }
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
{
ZDICT_params_t params;
memset(&params, 0, sizeof(params));
return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity,
samplesBuffer, samplesSizes, nbSamples,
params);
}

View File

@ -52,6 +52,21 @@ extern "C" {
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
/*! ZDICT_addEntropyTablesFromBuffer() :
Given a content-only dictionary (built for example from common strings in
the input), add entropy tables computed from the memory buffer
`samplesBuffer`, where `nbSamples` samples have been stored concatenated.
Each sample size is provided into an orderly table `samplesSizes`.
The input dictionary is the last `dictContentSize` bytes of `dictBuffer`. The
resulting dictionary with added entropy tables will written back to
`dictBuffer`.
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
*/
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
/*-************************************* /*-*************************************
* Helper functions * Helper functions
@ -60,6 +75,44 @@ unsigned ZDICT_isError(size_t errorCode);
const char* ZDICT_getErrorName(size_t errorCode); const char* ZDICT_getErrorName(size_t errorCode);
#ifdef ZDICT_STATIC_LINKING_ONLY
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
* ==================================================================================== */
/*-*************************************
* Public type
***************************************/
typedef struct {
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
unsigned reserved[2]; /* space for future parameters */
} ZDICT_params_t;
/*-*************************************
* Public functions
***************************************/
/*! ZDICT_trainFromBuffer_advanced() :
Same as ZDICT_trainFromBuffer() with control over more parameters.
`parameters` is optional and can be provided with values set to 0 to mean "default".
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
or an error code, which can be tested by ZDICT_isError().
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
*/
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
#endif /* ZDICT_STATIC_LINKING_ONLY */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

View File

@ -1,80 +0,0 @@
/*
dictBuilder header file
for static linking only
Copyright (C) Yann Collet 2016
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Zstd source repository : https://www.zstd.net
*/
/* This library is EXPERIMENTAL, below API is not yet stable */
#ifndef DICTBUILDER_STATIC_H_002
#define DICTBUILDER_STATIC_H_002
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Dependencies
***************************************/
#include "zdict.h"
/*-*************************************
* Public type
***************************************/
typedef struct {
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned reserved[3]; /* space for future parameters */
} ZDICT_params_t;
/*-*************************************
* Public functions
***************************************/
/*! ZDICT_trainFromBuffer_advanced() :
Same as ZDICT_trainFromBuffer() with control over more parameters.
`parameters` is optional and can be provided with values set to 0 to mean "default".
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
or an error code, which can be tested by DiB_isError().
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
*/
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t parameters);
#if defined (__cplusplus)
}
#endif
#endif /* DICTBUILDER_STATIC_H_002 */

View File

@ -1,3 +0,0 @@
-I../lib/common
-I../lib/legacy
-I./legacy

5
programs/.gitignore vendored
View File

@ -1,6 +1,8 @@
# local binary (Makefile) # local binary (Makefile)
zstd zstd
zstd32 zstd32
zstd-compress
zstd-decompress
fullbench fullbench
fullbench32 fullbench32
fuzzer fuzzer
@ -38,6 +40,9 @@ roundTripCrash
dictionary dictionary
grillResults.txt grillResults.txt
_* _*
tmp*
*.zst
result
# fuzzer # fuzzer
afl afl

View File

@ -33,29 +33,31 @@
DESTDIR?= DESTDIR?=
PREFIX ?= /usr/local PREFIX ?= /usr/local
CPPFLAGS= -I../lib/common
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
BINDIR = $(PREFIX)/bin BINDIR = $(PREFIX)/bin
MANDIR = $(PREFIX)/share/man/man1 MANDIR = $(PREFIX)/share/man/man1
ZSTDDIR = ../lib ZSTDDIR = ../lib
ZSTDCOMP_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/decompress/zstd_decompress.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES) ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
ZSTD_FILES_LEGACY:= ZSTDLEGACY_FILES:=
else else
ZSTD_LEGACY_SUPPORT:=1 ZSTD_LEGACY_SUPPORT:=1
CPPFLAGS += -I../lib/legacy -I./legacy CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \ ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c
$(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
endif endif
@ -78,12 +80,12 @@ default: zstd
all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32 all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32
zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \ zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
zstd_nolegacy : zstd_nolegacy :
@ -103,10 +105,12 @@ zstd-pgo : clean zstd
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-small: clean zstd-small: clean
@ -118,23 +122,21 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(ZSTD_FILES) \ fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
datagen.c xxhash.c fuzzer.c fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(ZSTD_FILES) \ fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
datagen.c xxhash.c fuzzer.c fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \ zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
datagen.c xxhash.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \ zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
datagen.c xxhash.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTD_FILES) datagen.c xxhash.c paramgrill.c paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT) $(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : datagen.c datagencli.c datagen : datagen.c datagencli.c
@ -206,7 +208,7 @@ endif
#------------------------------------------------------------------------ #------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX)) ifneq (,$(filter $(HOST_OS),MSYS POSIX))
zstd-playTests: datagen zstd-playTests: datagen
ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zbuff test: test-zstd test-fullbench test-fuzzer test-zbuff

View File

@ -32,7 +32,8 @@
#include <stdio.h> /* fprintf, fopen, ftello64 */ #include <stdio.h> /* fprintf, fopen, ftello64 */
#include "mem.h" #include "mem.h"
#include "zstd_static.h" #define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" #include "xxhash.h"
@ -220,7 +221,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ ZSTD_parameters params; { ZSTD_parameters params;
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
ZSTD_adjustCParams(&params.cParams, blockSize, dictBufferSize);
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
if (ZSTD_isError(initResult)) break; if (ZSTD_isError(initResult)) break;
} } } }
@ -505,4 +505,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
return 0; return 0;
} }

View File

@ -21,10 +21,11 @@
You can contact the author at : You can contact the author at :
- ZSTD homepage : http://www.zstd.net/ - ZSTD homepage : http://www.zstd.net/
*/ */
#pragma once #ifndef BENCH_H_121279284357
#define BENCH_H_121279284357
#include <stddef.h>
/* Main function */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast); const char* dictFileName, int cLevel, int cLevelLast);
@ -34,3 +35,4 @@ void BMK_SetBlockSize(size_t blockSize);
void BMK_setAdditionalParam(int additionalParam); void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level); void BMK_setNotificationLevel(unsigned level);
#endif /* BENCH_H_121279284357 */

View File

@ -101,27 +101,30 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo
/* ******************************************************** /* ********************************************************
* File related operations * File related operations
**********************************************************/ **********************************************************/
static void DiB_loadFiles(void* buffer, size_t bufferSize, /** DiB_loadFiles() :
* @return : nb of files effectively loaded into `buffer` */
static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes, size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles) const char** fileNamesTable, unsigned nbFiles)
{ {
char* buff = (char*)buffer; char* const buff = (char*)buffer;
size_t pos = 0; size_t pos = 0;
unsigned n; unsigned n;
for (n=0; n<nbFiles; n++) { for (n=0; n<nbFiles; n++) {
size_t readSize; unsigned long long const fs64 = UTIL_getFileSize(fileNamesTable[n]);
unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]); size_t const fileSize = (size_t)(fs64 > bufferSize-pos ? 0 : fs64);
FILE* f = fopen(fileNamesTable[n], "rb"); FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */ { size_t const readSize = fread(buff+pos, 1, fileSize, f);
readSize = fread(buff+pos, 1, (size_t)fileSize, f); if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); pos += readSize; }
pos += readSize; fileSizes[n] = fileSize;
fileSizes[n] = (size_t)fileSize;
fclose(f); fclose(f);
if (fileSize == 0) break; /* stop there, not enough memory to load all files */
} }
return n;
} }
@ -130,7 +133,7 @@ static void DiB_loadFiles(void* buffer, size_t bufferSize,
**********************************************************/ **********************************************************/
static size_t DiB_findMaxMem(unsigned long long requiredMem) static size_t DiB_findMaxMem(unsigned long long requiredMem)
{ {
size_t step = 8 MB; size_t const step = 8 MB;
void* testmem = NULL; void* testmem = NULL;
requiredMem = (((requiredMem >> 23) + 1) << 23); requiredMem = (((requiredMem >> 23) + 1) << 23);
@ -162,7 +165,7 @@ static void DiB_fillNoise(void* buffer, size_t length)
static void DiB_saveDict(const char* dictFileName, static void DiB_saveDict(const char* dictFileName,
const void* buff, size_t buffSize) const void* buff, size_t buffSize)
{ {
FILE* f = fopen(dictFileName, "wb"); FILE* const f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
{ size_t const n = fwrite(buff, 1, buffSize, f); { size_t const n = fwrite(buff, 1, buffSize, f);
@ -185,35 +188,32 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
ZDICT_params_t parameters); ZDICT_params_t parameters);
#define MIN(a,b) ((a)<(b)?(a):(b))
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles, const char** fileNamesTable, unsigned nbFiles,
ZDICT_params_t params) ZDICT_params_t params)
{ {
void* srcBuffer; void* const dictBuffer = malloc(maxDictSize);
size_t benchedSize; size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
unsigned long long totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
void* dictBuffer = malloc(maxDictSize); size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
size_t dictSize; void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
int result = 0; int result = 0;
/* Checks */
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
/* init */ /* init */
g_displayLevel = params.notificationLevel; g_displayLevel = params.notificationLevel;
benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad) if (benchedSize < totalSizeToLoad)
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
/* Memory allocation & restrictions */
srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
/* Load input buffer */ /* Load input buffer */
DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
/* call buffer version */ { size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
srcBuffer, fileSizes, nbFiles, srcBuffer, fileSizes, nbFiles,
params); params);
if (ZDICT_isError(dictSize)) { if (ZDICT_isError(dictSize)) {
@ -221,10 +221,10 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
result = 1; result = 1;
goto _cleanup; goto _cleanup;
} }
/* save dict */ /* save dict */
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
DiB_saveDict(dictFileName, dictBuffer, dictSize); DiB_saveDict(dictFileName, dictBuffer, dictSize);
}
/* clean up */ /* clean up */
_cleanup: _cleanup:

View File

@ -32,7 +32,8 @@
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */ #define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h" /* ZDICT_params_t */
/*-************************************* /*-*************************************

View File

@ -56,9 +56,11 @@
#include "mem.h" #include "mem.h"
#include "fileio.h" #include "fileio.h"
#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h"
#include "zstd_internal.h" /* MIN, KB, MB */ #include "zstd_internal.h" /* MIN, KB, MB */
#include "zbuff_static.h" #define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* ZSTD_isLegacy */ # include "zstd_legacy.h" /* ZSTD_isLegacy */
@ -97,7 +99,7 @@
#define CACHELINE 64 #define CACHELINE 64
#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */ #define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30 #define FNSPACE 30
@ -133,6 +135,10 @@ static U32 g_maxWLog = 23;
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
static U32 g_checksumFlag = 0;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
/*-************************************* /*-*************************************
@ -186,7 +192,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
} }
} else { } else {
if (!g_overwrite) { /* Check if destination file already exists */ if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
f = fopen( dstFileName, "rb" ); f = fopen( dstFileName, "rb" );
if (f != 0) { /* dest file exists, prompt for overwrite authorization */ if (f != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(f); fclose(f);
@ -212,12 +218,12 @@ static FILE* FIO_openDstFile(const char* dstFileName)
/*! FIO_loadFile() : /*! FIO_loadFile() :
* creates a buffer, pointed by `*bufferPtr`, * creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, * loads `filename` content into it,
* up to MAX_DICT_SIZE bytes * up to MAX_DICT_SIZE bytes.
* @return : loaded size
*/ */
static size_t FIO_loadFile(void** bufferPtr, const char* fileName) static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
{ {
FILE* fileHandle; FILE* fileHandle;
size_t readSize;
U64 fileSize; U64 fileSize;
*bufferPtr = NULL; *bufferPtr = NULL;
@ -237,8 +243,8 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
} }
*bufferPtr = (BYTE*)malloc((size_t)fileSize); *bufferPtr = (BYTE*)malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer"); if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
fclose(fileHandle); fclose(fileHandle);
return (size_t)fileSize; return (size_t)fileSize;
} }
@ -308,12 +314,18 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */ /* init */
{ ZSTD_parameters params; { ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize); params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; params.fParams.checksumFlag = g_checksumFlag;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); params.fParams.noDictIDFlag = !g_dictIDFlag;
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); } if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) {
params.cParams.windowLog = g_maxWLog;
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
} }
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
} }
/* Main compression loop */ /* Main compression loop */
readsize = 0; readsize = 0;
@ -409,14 +421,10 @@ static int FIO_compressFilename_extRess(cRess_t ress,
int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel) const char* dictFileName, int compressionLevel)
{ {
clock_t start; clock_t const start = clock();
cRess_t ress; cRess_t const ress = FIO_createCResources(dictFileName);
int issueWithSrcFile = 0; int issueWithSrcFile = 0;
/* Init */
start = clock();
ress = FIO_createCResources(dictFileName);
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel); issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
FIO_freeCResources(ress); FIO_freeCResources(ress);
@ -607,7 +615,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Header loading (optional, saves one loop) */ /* Header loading (optional, saves one loop) */
{ size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */ { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
readSize = alreadyLoaded + loadedSize; readSize = alreadyLoaded + loadedSize;
} }
@ -689,7 +697,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
continue; continue;
} }
#endif #endif
if (magic != ZSTD_MAGICNUMBER) { if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
if (g_overwrite) /* -df : pass-through mode */ if (g_overwrite) /* -df : pass-through mode */
return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
else { else {
@ -701,7 +709,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
/* Final Status */ /* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
/* Close */ /* Close */
fclose(srcFile); fclose(srcFile);

View File

@ -48,6 +48,8 @@ void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level); void FIO_setNotificationLevel(unsigned level);
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
/*-************************************* /*-*************************************

View File

@ -28,12 +28,13 @@
#include "util.h" /* Compiler options, UTIL_GetFileSize */ #include "util.h" /* Compiler options, UTIL_GetFileSize */
#include <stdlib.h> /* malloc */ #include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen, ftello64 */ #include <stdio.h> /* fprintf, fopen, ftello64 */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "mem.h" #include "mem.h"
#include "zstd_static.h" /* ZSTD_VERSION_STRING */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */
#include "fse_static.h" #include "zstd.h" /* ZSTD_VERSION_STRING */
#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */
#include "fse.h"
#include "zbuff.h" #include "zbuff.h"
#include "datagen.h" #include "datagen.h"
@ -544,4 +545,3 @@ int main(int argc, const char** argv)
return result; return result;
} }

View File

@ -28,7 +28,7 @@
#ifdef _MSC_VER /* Visual Studio */ #ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */ # define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif #endif
@ -40,7 +40,9 @@
#include <sys/timeb.h> /* timeb */ #include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#include <time.h> /* clock_t */ #include <time.h> /* clock_t */
#include "zstd_static.h" /* ZSTD_VERSION_STRING */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */
#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "mem.h" #include "mem.h"
#define XXH_STATIC_LINKING_ONLY #define XXH_STATIC_LINKING_ONLY
@ -54,7 +56,6 @@
#define MB *(1U<<20) #define MB *(1U<<20)
#define GB *(1U<<30) #define GB *(1U<<30)
static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */
static const U32 FUZ_compressibility_default = 50; static const U32 FUZ_compressibility_default = 50;
static const U32 nbTestsDefault = 30000; static const U32 nbTestsDefault = 30000;
@ -78,7 +79,6 @@ static clock_t g_displayClock = 0;
* Fuzzer functions * Fuzzer functions
*********************************************************/ *********************************************************/
#define MIN(a,b) ((a)<(b)?(a):(b)) #define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
static clock_t FUZ_clockSpan(clock_t cStart) static clock_t FUZ_clockSpan(clock_t cStart)
{ {
@ -86,8 +86,8 @@ static clock_t FUZ_clockSpan(clock_t cStart)
} }
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src) static unsigned FUZ_rand(unsigned* src)
{ {
static const U32 prime1 = 2654435761U; static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U; static const U32 prime2 = 2246822519U;
@ -104,142 +104,131 @@ static unsigned FUZ_highbit32(U32 v32)
{ {
unsigned nbBits = 0; unsigned nbBits = 0;
if (v32==0) return 0; if (v32==0) return 0;
while (v32) { while (v32) v32 >>= 1, nbBits++;
v32 >>= 1;
nbBits ++;
}
return nbBits; return nbBits;
} }
#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
#define CHECK(fn) { CHECKTEST(err, fn); }
#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; }
static int basicUnitTests(U32 seed, double compressibility) static int basicUnitTests(U32 seed, double compressibility)
{ {
size_t const CNBuffSize = 5 MB;
void* const CNBuffer = malloc(CNBuffSize);
void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
void* const decodedBuffer = malloc(CNBuffSize);
int testResult = 0; int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0; U32 testNb=0;
size_t cSize;
/* Create compressible test buffer */ /* Create compressible noise */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer) { if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory, aborting\n"); DISPLAY("Not enough memory, aborting\n");
testResult = 1; testResult = 1;
goto _end; goto _end;
} }
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState); RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
/* Basic tests */ /* Basic tests */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
if (ZSTD_isError(result)) goto _output_error; CNBuffer, CNBuffSize, 1),
cSize = result; cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize),
if (ZSTD_isError(result)) goto _output_error; if (r != CNBuffSize) goto _output_error);
if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) { { size_t u;
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; for (u=0; u<CNBuffSize; u++) {
} if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(r)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(r)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* Dictionary and CCtx Duplication tests */ /* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* ctxOrig = ZSTD_createCCtx(); { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const dictSize = 500; static const size_t dictSize = 551;
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++); DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error should be detected */ if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
{ size_t const initResult = ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2); CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
if (ZSTD_isError(initResult)) goto _output_error; } CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(copyResult)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
cSize = 0; cSize = 0;
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
if (ZSTD_isError(result)) goto _output_error; (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize += result; cSize += r);
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
if (ZSTD_isError(result)) goto _output_error; cSize += r);
cSize += result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx, CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, decodedBuffer, CNBuffSize,
compressedBuffer, cSize, compressedBuffer, cSize,
CNBuffer, dictSize); CNBuffer, dictSize),
if (ZSTD_isError(result)) goto _output_error; if (r != CNBuffSize - dictSize) goto _output_error);
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
{ size_t const cSizeOrig = cSize; { size_t const cSizeOrig = cSize;
cSize = 0; cSize = 0;
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
if (ZSTD_isError(result)) goto _output_error; (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize += result; cSize += r);
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
if (ZSTD_isError(result)) goto _output_error; cSize += r);
cSize += result; if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> have same size */
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
} }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx, CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, decodedBuffer, CNBuffSize,
compressedBuffer, cSize, compressedBuffer, cSize,
CNBuffer, dictSize); CNBuffer, dictSize),
if (ZSTD_isError(result)) goto _output_error; if (r != CNBuffSize - dictSize) goto _output_error);
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3; { size_t const testSize = CNBuffSize / 3;
{ ZSTD_parameters p; { ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
p.cParams = ZSTD_getCParams(2, testSize, dictSize); ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
p.fParams.contentSizeFlag = 1; ZSTD_parameters p;
{ size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1); p.cParams = cPar; p.fParams = fPar;
if (ZSTD_isError(initResult)) goto _output_error; CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
} } }
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
if (ZSTD_isError(copyResult)) goto _output_error; }
cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
if (ZSTD_isError(cSize)) goto _output_error; (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize = r);
{ ZSTD_frameParams fp; { ZSTD_frameParams fp;
size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize); if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
if (gfpResult!=0) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error; if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
} } } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@ -249,55 +238,113 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
} }
/* Dictionary and dictBuilder tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t dictSize = 16 KB;
void* dictBuffer = malloc(dictSize);
size_t const totalSampleSize = 1 MB;
size_t const sampleUnitSize = 8 KB;
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
if (dictBuffer==NULL || samplesSizes==NULL) {
free(dictBuffer);
free(samplesSizes);
goto _output_error;
}
DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
CNBuffer, samplesSizes, nbSamples);
if (ZDICT_isError(dictSize)) goto _output_error;
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize,
dictBuffer, dictSize, 4);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
ZSTD_parameters p;
p.cParams = cParams; p.fParams = fParams;
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize,
dictBuffer, dictSize, p);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(dictBuffer);
free(samplesSizes);
}
/* Decompression defense tests */ /* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(r)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1; ((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* block API tests */ /* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx(); { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB; static const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB; static const size_t dictSize = 16 KB;
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5); CHECK( ZSTD_compressBegin(cctx, 5) );
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_decompressBegin(dctx); CHECK( ZSTD_decompressBegin(dctx) );
if (ZSTD_isError(result)) goto _output_error; { CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (r != blockSize) goto _output_error; }
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */ /* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
result = ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5); CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize); CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
if (ZSTD_isError(result)) goto _output_error; { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (r != blockSize) goto _output_error; }
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
@ -307,32 +354,29 @@ static int basicUnitTests(U32 seed, double compressibility)
/* long rle test */ /* long rle test */
{ size_t sampleSize = 0; { size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState); RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1; sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState); RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
sampleSize += 96 KB; sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); { CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (ZSTD_isError(result)) goto _output_error; if (regenSize!=sampleSize) goto _output_error; }
if (result!=sampleSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
} }
/* All zeroes test (#137 verif) */ /* All zeroes test (test bug #137) */
#define ZEROESLENGTH 100 #define ZEROESLENGTH 100
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
memset(CNBuffer, 0, ZEROESLENGTH); memset(CNBuffer, 0, ZEROESLENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1); { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
if (ZSTD_isError(result)) goto _output_error; cSize = r; }
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
result = ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize); { CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
if (ZSTD_isError(result)) goto _output_error; if (r != ZEROESLENGTH) goto _output_error; }
if (result != ZEROESLENGTH) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* nbSeq limit test */ /* nbSeq limit test */
@ -352,23 +396,21 @@ static int basicUnitTests(U32 seed, double compressibility)
}} }}
/* randomly fills CNBuffer with prepared 3-bytes sequences */ /* randomly fills CNBuffer with prepared 3-bytes sequences */
{ int i; for (i=0; i < _3BYTESTESTLENGTH; ) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
U32 id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
i += 3;
} }} } }}
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19); { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
if (ZSTD_isError(result)) goto _output_error; CNBuffer, _3BYTESTESTLENGTH, 19) );
cSize = result; cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
result = ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize); { CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
if (ZSTD_isError(result)) goto _output_error; if (r != _3BYTESTESTLENGTH) goto _output_error; }
if (result != _3BYTESTESTLENGTH) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
_end: _end:
@ -388,11 +430,11 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{ {
const BYTE* b1 = (const BYTE*)buf1; const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2; const BYTE* b2 = (const BYTE*)buf2;
size_t i; size_t u;
for (i=0; i<max; i++) { for (u=0; u<max; u++) {
if (b1[i] != b2[i]) break; if (b1[u] != b2[u]) break;
} }
return i; return u;
} }
@ -408,6 +450,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
return FUZ_rLogLength(seed, logLength); return FUZ_rLogLength(seed, logLength);
} }
#undef CHECK
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
@ -415,35 +458,29 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
{ {
static const U32 maxSrcLog = 23; static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22; static const U32 maxSampleLog = 22;
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
BYTE* cNoiseBuffer[5]; BYTE* cNoiseBuffer[5];
BYTE* srcBuffer; BYTE* srcBuffer; /* jumping pointer */
BYTE* cBuffer; BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
BYTE* dstBuffer; BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
BYTE* mirrorBuffer; BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
size_t srcBufferSize = (size_t)1<<maxSrcLog; ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
size_t dstBufferSize = (size_t)1<<maxSampleLog; ZSTD_CCtx* const ctx = ZSTD_createCCtx();
size_t cBufferSize = ZSTD_compressBound(dstBufferSize); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
U32 result = 0; U32 result = 0;
U32 testNb = 0; U32 testNb = 0;
U32 coreSeed = seed, lseed = 0; U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* refCtx; clock_t const startClock = clock();
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
clock_t startClock = clock();
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC; clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
/* allocation */ /* allocation */
refCtx = ZSTD_createCCtx();
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled"); "Not enough memory, fuzzer tests cancelled");
@ -461,9 +498,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* main test loop */ /* main test loop */
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) { for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t sampleSize, maxTestSize, totalTestSize;
size_t cSize, dSize, totalCSize, totalGenSize; size_t cSize, totalCSize, totalGenSize;
U32 sampleSizeLog, nbChunks, n;
XXH64_state_t xxhState; XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
BYTE* sampleBuffer; BYTE* sampleBuffer;
@ -493,18 +529,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
} }
/* select src segment */ /* select src segment */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
/* create sample buffer (to catch read error with valgrind & sanitizers) */ /* create sample buffer (to catch read error with valgrind & sanitizers) */
sampleBuffer = (BYTE*)malloc(sampleSize); sampleBuffer = (BYTE*)malloc(sampleSize);
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
crcOrig = XXH64(sampleBuffer, sampleSize, 0); crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression tests */ /* compression tests */
{ int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1; { unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
@ -518,9 +553,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); } CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4); { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
} } }
}
/* frame header decompression test */ /* frame header decompression test */
{ ZSTD_frameParams dParams; { ZSTD_frameParams dParams;
@ -531,7 +564,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* successful decompression test */ /* successful decompression test */
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
@ -542,7 +575,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* truncated src decompression test */ /* truncated src decompression test */
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = cSize - missing; size_t const tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !"); CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize); memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
@ -575,13 +608,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
} }
if (pos <= cSize) break; if (pos <= cSize) break;
/* add noise */ /* add noise */
{ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
size_t mask, noiseStart, noiseLength; U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
if (nbBits>0) nbBits--; size_t const mask = (1<<nbBits) - 1;
mask = (1<<nbBits) - 1; size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1; size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength; pos += noiseLength;
} } } } } }
@ -605,35 +637,45 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
maxTestSize = FUZ_rLogLength(&lseed, testLog); maxTestSize = FUZ_rLogLength(&lseed, testLog);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSize = FUZ_randomLength(&lseed, maxSampleLog); dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
{ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); if (FUZ_rand(&lseed) & 0xF) {
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_parameters p;
size_t errorCode;
p.cParams = cPar; p.fParams = fpar;
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
} }
XXH64_reset(&xxhState, 0); XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
U32 n;
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
sampleSize = (size_t)1 << sampleSizeLog; size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) break; /* avoid invalid dstBufferTooSmall */ if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
if (totalTestSize+sampleSize > maxTestSize) break; if (totalTestSize+segmentSize > maxTestSize) break;
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
cSize += compressResult; cSize += compressResult;
} }
XXH64_update(&xxhState, srcBuffer+sampleStart, sampleSize); XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
totalTestSize += sampleSize; totalTestSize += segmentSize;
} } }
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult; cSize += flushResult;
@ -641,8 +683,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
crcOrig = XXH64_digest(&xxhState); crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */ /* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); } CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
totalCSize = 0; totalCSize = 0;
totalGenSize = 0; totalGenSize = 0;
while (totalCSize < cSize) { while (totalCSize < cSize) {
@ -653,12 +696,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
totalCSize += inSize; totalCSize += inSize;
} }
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read") CHECK (totalCSize != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) { if (crcDest!=crcOrig) {
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize); size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]); (U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
} } } }
} /* for ( ; (testNb <= nbTests) */ } /* for ( ; (testNb <= nbTests) */

View File

@ -48,7 +48,8 @@
#endif #endif
#include "mem.h" #include "mem.h"
#include "zstd_static.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "zstd.h"
#include "datagen.h" #include "datagen.h"
#include "xxhash.h" #include "xxhash.h"
@ -600,22 +601,22 @@ static void playAround(FILE* f, winnerInfo_t* winners,
} }
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance) static ZSTD_compressionParameters randomParams(void)
{ {
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1)); ZSTD_compressionParameters p;
U32 validated = 0; U32 validated = 0;
if (!chance)
while (!validated) { while (!validated) {
/* totally random entry */ /* totally random entry */
p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
validated = !ZSTD_isError(ZSTD_checkCParams(*p)); validated = !ZSTD_isError(ZSTD_checkCParams(p));
} }
return p;
} }
static void BMK_selectRandomStart( static void BMK_selectRandomStart(
@ -623,12 +624,10 @@ static void BMK_selectRandomStart(
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx) ZSTD_CCtx* ctx)
{ {
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
if ((id==0) || (winners[id].params.windowLog==0)) { if ((id==0) || (winners[id].params.windowLog==0)) {
/* totally random entry */ /* totally random entry */
ZSTD_compressionParameters p; ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
potentialRandomParams(&p, 1);
ZSTD_adjustCParams(&p, srcSize, 0);
playAround(f, winners, p, srcBuffer, srcSize, ctx); playAround(f, winners, p, srcBuffer, srcSize, ctx);
} }
else else
@ -649,7 +648,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun) { if (g_singleRun) {
BMK_result_t testResult; BMK_result_t testResult;
ZSTD_adjustCParams(&g_params, srcSize, 0); g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n"); DISPLAY("\n");
return; return;
@ -861,7 +860,7 @@ int optimizeForSize(char* inFileName)
do { do {
params = winner.params; params = winner.params;
paramVariation(&params); paramVariation(&params);
potentialRandomParams(&params, 16); if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams();
/* exclude faster if already played set of params */ /* exclude faster if already played set of params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;

View File

@ -15,22 +15,30 @@ roundTripTest() {
rm -f tmp1 tmp2 rm -f tmp1 tmp2
$ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
./datagen $1 $p | md5sum > tmp1 ./datagen $1 $p | $MD5SUM > tmp1
./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2
diff -q tmp1 tmp2 diff -q tmp1 tmp2
} }
isWindows=false isWindows=false
ECHO="echo" ECHO="echo"
if [[ "$OS" == "Windows"* ]]; then case "$OS" in
Windows*)
isWindows=true isWindows=true
ECHO="echo -e" ECHO="echo -e"
;;
esac
MD5SUM="md5sum"
if [ "$TRAVIS_OS_NAME" = "osx" ]; then
MD5SUM="md5 -r"
fi fi
$ECHO "\nStarting playTests.sh isWindows=$isWindows" $ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
file $ZSTD
$ECHO "\n**** simple tests **** " $ECHO "\n**** simple tests **** "
./datagen > tmp ./datagen > tmp
@ -114,19 +122,32 @@ $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated $ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse* ls -ls tmpSparse*
diff tmpSparse2M tmpSparseRegenerated diff tmpSparse2M tmpSparseRegenerated
# rm tmpSparse* rm tmpSparse*
$ECHO "\n**** dictionary tests **** " $ECHO "\n**** dictionary tests **** "
./datagen > tmpDict ./datagen > tmpDict
./datagen -g1M | md5sum > tmp1 ./datagen -g1M | $MD5SUM > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
diff -q tmp1 tmp2 diff -q tmp1 tmp2
$ZSTD --train *.c *.h -o tmpDict $ECHO "Create first dictionary"
$ZSTD xxhash.c -D tmpDict -of tmp $ZSTD --train *.c -o tmpDict
$ZSTD -d tmp -D tmpDict -of result cp zstdcli.c tmp
diff xxhash.c result $ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -of result
diff zstdcli.c result
$ECHO "Create second (different) dictionary"
$ZSTD --train *.c *.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
$ECHO "Create dictionary with short dictID"
$ZSTD --train *.c --dictID 1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "Compress without dictID"
$ZSTD -f tmp -D tmpDict1 --no-dictID
$ZSTD -d tmp.zst -D tmpDict -of result
diff zstdcli.c result
rm tmp*
$ECHO "\n**** multiple files tests **** " $ECHO "\n**** multiple files tests **** "
@ -210,4 +231,3 @@ roundTripTest -g99000000 -P99 20
roundTripTest -g6000000000 -P99 1 roundTripTest -g6000000000 -P99 1
rm tmp* rm tmp*

View File

@ -1,157 +0,0 @@
@echo off
ECHO Start playTests.bat
ECHO ZSTD^: %ZSTD%
ECHO DATAGEN^: %DATAGEN%
if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1
if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1
SET ROUNDTRIPTEST=tests\roundTripTest.bat
echo. && echo **** simple tests ****
%DATAGEN% > tmp
%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst
%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp)
echo test : too large compression level (must fail)
%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1)
echo test : compress to stdout
%ZSTD% tmp -c > tmpCompressed
%ZSTD% tmp --stdout > tmpCompressed && REM long command format
echo test : null-length file roundtrip
echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1)
echo test : decompress file with wrong suffix (must fail)
%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1)
%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout
%ZSTD% --decompress tmpCompressed -c > tmpResult
%ZSTD% --decompress tmpCompressed --stdout > tmpResult
REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout
REM %ZSTD% -d - < tmp.zst > NUL
%ZSTD% -dc < tmp.zst > NUL
%ZSTD% -dc - < tmp.zst > NUL
%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1)
%ZSTD% -q -f tmp
%ZSTD% -q --force tmp
%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1)
echo. && echo **** Pass-Through mode ****
echo "Hello world !" | %ZSTD% -df
echo "Hello world !" | %ZSTD% -dcf
echo. && echo **** frame concatenation ****
echo hello > hello.tmp
echo world! > world.tmp
cat hello.tmp world.tmp > helloworld.tmp
%ZSTD% -c hello.tmp > hello.zstd
%ZSTD% -c world.tmp > world.zstd
cat hello.zstd world.zstd > helloworld.zstd
%ZSTD% -dc helloworld.zstd > result.tmp
cat result.tmp
fc /b helloworld.tmp result.tmp
rm *.tmp *.zstd
echo frame concatenation tests completed
REM echo. && echo **** flush write error test ****
REM echo echo foo ^| %ZSTD% ^> v:\full
REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1)
REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full"
REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1)
echo. && echo **** dictionary tests ****
%DATAGEN% > tmpDict
%DATAGEN% -g1M | md5sum > tmp1
%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2
fc tmp1 tmp2
%ZSTD% --train *.c *.h -o tmpDict
%ZSTD% xxhash.c -D tmpDict -of tmp
%ZSTD% -d tmp -D tmpDict -of result
fc xxhash.c result
echo. && echo **** multiple files tests ****
%DATAGEN% -s1 > tmp1 2> NUL
%DATAGEN% -s2 -g100K > tmp2 2> NUL
%DATAGEN% -s3 -g1M > tmp3 2> NUL
%ZSTD% -f tmp*
echo compress tmp* :
ls -ls tmp*
rm tmp1 tmp2 tmp3
echo decompress tmp* :
%ZSTD% -df *.zst
ls -ls tmp*
echo compress tmp* into stdout ^> tmpall :
%ZSTD% -c tmp1 tmp2 tmp3 > tmpall
ls -ls tmp*
echo decompress tmpall* into stdout ^> tmpdec :
cp tmpall tmpall2
%ZSTD% -dc tmpall* > tmpdec
ls -ls tmp*
echo compress multiple files including a missing one (notHere) :
%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1)
echo. && echo **** integrity tests ****
echo test one file (tmp1.zst)
%ZSTD% -t tmp1.zst
%ZSTD% --test tmp1.zst
echo test multiple files (*.zst)
%ZSTD% -t *.zst
echo test good and bad files (*)
%ZSTD% -t * && (echo bad files not detected! && exit /b 1)
echo. && echo **** zstd round-trip tests ****
CALL %ROUNDTRIPTEST%
CALL %ROUNDTRIPTEST% -g15K && REM TableID==3
CALL %ROUNDTRIPTEST% -g127K && REM TableID==2
CALL %ROUNDTRIPTEST% -g255K && REM TableID==1
CALL %ROUNDTRIPTEST% -g513K && REM TableID==0
CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain
CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2
CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt
rm tmp*
echo Param = %1
if NOT "%1"=="--test-large-data" (
echo skipping large data tests
exit /b 0
)
CALL %ROUNDTRIPTEST% -g270000000 1
CALL %ROUNDTRIPTEST% -g270000000 2
CALL %ROUNDTRIPTEST% -g270000000 3
CALL %ROUNDTRIPTEST% -g140000000 -P60 4
CALL %ROUNDTRIPTEST% -g140000000 -P60 5
CALL %ROUNDTRIPTEST% -g140000000 -P60 6
CALL %ROUNDTRIPTEST% -g70000000 -P70 7
CALL %ROUNDTRIPTEST% -g70000000 -P70 8
CALL %ROUNDTRIPTEST% -g70000000 -P70 9
CALL %ROUNDTRIPTEST% -g35000000 -P75 10
CALL %ROUNDTRIPTEST% -g35000000 -P75 11
CALL %ROUNDTRIPTEST% -g35000000 -P75 12
CALL %ROUNDTRIPTEST% -g18000000 -P80 13
CALL %ROUNDTRIPTEST% -g18000000 -P80 14
CALL %ROUNDTRIPTEST% -g18000000 -P80 15
CALL %ROUNDTRIPTEST% -g18000000 -P80 16
CALL %ROUNDTRIPTEST% -g18000000 -P80 17
CALL %ROUNDTRIPTEST% -g50000000 -P94 18
CALL %ROUNDTRIPTEST% -g50000000 -P94 19
CALL %ROUNDTRIPTEST% -g99000000 -P99 20
CALL %ROUNDTRIPTEST% -g6000000000 -P99 1
rm tmp*
exit /b 0

View File

@ -1,8 +0,0 @@
@echo off
if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2)
rm -f tmp1 tmp2
echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d
%DATAGEN% %1 %P% | md5sum > tmp1
%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2
fc tmp1 tmp2
EXIT /B %ERRORLEVEL%

View File

@ -40,12 +40,13 @@
#include <sys/timeb.h> /* timeb */ #include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#include "mem.h" #include "mem.h"
#include "zbuff.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ #include "zstd.h" /* ZSTD_compressBound */
#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY #define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */ #include "xxhash.h" /* XXH64_* */
/*-************************************ /*-************************************
@ -97,7 +98,7 @@ static U32 FUZ_GetMilliStart(void)
static U32 FUZ_GetMilliSpan(U32 nTimeStart) static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{ {
U32 nCurrent = FUZ_GetMilliStart(); U32 const nCurrent = FUZ_GetMilliStart();
U32 nSpan = nCurrent - nTimeStart; U32 nSpan = nCurrent - nTimeStart;
if (nTimeStart > nCurrent) if (nTimeStart > nCurrent)
nSpan += 0x100000 * 1000; nSpan += 0x100000 * 1000;
@ -129,16 +130,18 @@ static unsigned FUZ_highbit32(U32 v32)
} }
*/ */
void* ZBUFF_allocFunction(size_t size) static void* ZBUFF_allocFunction(void* opaque, size_t size)
{ {
void* address = malloc(size); void* address = malloc(size);
/* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */ (void)opaque;
/* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
return address; return address;
} }
void ZBUFF_freeFunction(void* address) static void ZBUFF_freeFunction(void* opaque, void* address)
{ {
/* if (address) DISPLAYLEVEL(4, "free %p \n", address); */ (void)opaque;
/* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */
free(address); free(address);
} }
@ -147,11 +150,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
int testResult = 0; int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize); void* CNBuffer = malloc(CNBufferSize);
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize); void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize; size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize); void* decodedBuffer = malloc(decodedBufferSize);
size_t result, cSize, readSize, genSize; size_t cSize, readSize, readSkipSize, genSize;
U32 testNb=0; U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
@ -163,45 +167,61 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
} }
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
readSize = CNBufferSize; readSize = CNBufferSize;
genSize = compressedBufferSize; genSize = compressedBufferSize;
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error; if (ZBUFF_isError(r)) goto _output_error; }
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
cSize = genSize; cSize += genSize;
genSize = compressedBufferSize - cSize; genSize = compressedBufferSize - cSize;
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (result != 0) goto _output_error; /* error, or some data not flushed */ if (r != 0) goto _output_error; } /*< error, or some data not flushed */
cSize += genSize; cSize += genSize;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* skippable frame test */
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSkipSize = cSize;
genSize = CNBufferSize;
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
if (r != 0) goto _output_error; }
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
DISPLAYLEVEL(4, "OK \n");
/* Basic decompression test */ /* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSize = cSize; readSize = cSize - readSkipSize;
genSize = CNBufferSize; genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */ if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */ /* check regenerated data is byte exact */
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) { for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
} } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
}
/* Byte-by-byte decompression test */ /* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); { size_t r, pIn=0, pOut=0;
{ size_t r = 1, pIn=0, pOut=0; do
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
r = 1;
while (r) { while (r) {
size_t inS = 1; size_t inS = 1;
size_t outS = 1; size_t outS = 1;
@ -211,19 +231,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
} }
readSize = pIn; readSize = pIn;
genSize = pOut; genSize = pOut;
} while (genSize==0);
} }
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */ /* check regenerated data is byte exact */
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) { for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
} } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
}
_end: _end:
ZBUFF_freeCCtx(zc); ZBUFF_freeCCtx(zc);
@ -332,8 +352,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed); FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1; lseed = coreSeed ^ prime1;
/* state total reset */ /* states full reset (unsynchronized) */
/* some problems only happen when states are re-used in a specific order */ /* some issues only happen when reusing states in a specific sequence of parameters */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
@ -356,15 +376,21 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog); maxTestSize = FUZ_rLogLength(&lseed, testLog);
/* random dictionary selection */
{ size_t dictStart;
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); /* random dictionary selection */
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart; dict = srcBuffer + dictStart;
} }
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel); { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
U32 const checksum = FUZ_rand(&lseed) & 1;
U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
ZSTD_parameters params;
params.cParams = cPar;
params.fParams = fPar;
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} } } } }
/* multi-segments compression test */ /* multi-segments compression test */
XXH64_reset(&xxhState, 0); XXH64_reset(&xxhState, 0);
@ -498,8 +524,8 @@ int main(int argc, const char** argv)
int result=0; int result=0;
U32 mainPause = 0; U32 mainPause = 0;
const char* programName = argv[0]; const char* programName = argv[0];
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction }; ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL };
ZSTD_customMem customNULL = { NULL, NULL }; ZSTD_customMem customNULL = { NULL, NULL, NULL };
/* Check command line */ /* Check command line */
for(argNb=1; argNb<argc; argNb++) { for(argNb=1; argNb<argc; argNb++) {
@ -600,9 +626,10 @@ int main(int argc, const char** argv)
if (testNb==0) { if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */ result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result) if (!result) {
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} } }
if (!result) if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);

View File

@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
use `file` as Dictionary to compress or decompress FILE(s) use `file` as Dictionary to compress or decompress FILE(s)
.TP .TP
.B \-o file .B \-o file
save result into `file` (only possible with a single input FILE) save result into `file` (only possible with a single INPUT-FILE)
.TP .TP
.BR \-f ", " --force .BR \-f ", " --force
overwrite output without prompting overwrite output without prompting
@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
.TP .TP
.BR \-c ", " --stdout .BR \-c ", " --stdout
force write to standard output, even if it is the console force write to standard output, even if it is the console
.TP
.BR \-C ", " --check
add integrity check computed from uncompressed data
.TP
.BR \-t ", " --test
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
No files are created or removed.
.SH DICTIONARY .SH DICTIONARY
.PP .PP
@ -92,6 +99,14 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
.B \--maxdict # .B \--maxdict #
limit dictionary to specified size (default : 112640) limit dictionary to specified size (default : 112640)
.TP .TP
.B \--dictID #
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
By default, zstd will create a 4-bytes random number ID.
It's possible to give a precise number instead.
Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
.TP
.B \-s# .B \-s#
dictionary selectivity level (default: 9) dictionary selectivity level (default: 9)
the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size. the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.

View File

@ -38,10 +38,11 @@
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ # include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
#endif #endif
#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
# include "dibio.h" # include "dibio.h"
#endif #endif
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_VERSION_STRING */
@ -52,7 +53,7 @@
# include <io.h> /* _isatty */ # include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else #else
#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE #if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
# include <unistd.h> /* isatty */ # include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else #else
@ -125,7 +126,6 @@ static int usage_advanced(const char* programName)
DISPLAY( "\n"); DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n"); DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n"); DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -t : test compressed file integrity \n");
DISPLAY( " -v : verbose mode\n"); DISPLAY( " -v : verbose mode\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n");
@ -134,8 +134,13 @@ static int usage_advanced(const char* programName)
#endif #endif
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
DISPLAY( "--check : enable integrity check\n");
#endif #endif
#ifndef ZSTD_NODECOMPRESS
DISPLAY( "--test : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
#endif
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
DISPLAY( "\n"); DISPLAY( "\n");
DISPLAY( "Dictionary builder :\n"); DISPLAY( "Dictionary builder :\n");
@ -143,6 +148,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n");
#endif #endif
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
DISPLAY( "\n"); DISPLAY( "\n");
@ -171,6 +177,18 @@ static void waitEnter(void)
(void)unused; (void)unused;
} }
/*! readU32FromChar() :
@return : unsigned integer value reach from input in `char` format
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : this function can overflow if result > MAX_UNIT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
return result;
}
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@ -185,7 +203,8 @@ int main(int argCount, const char** argv)
operationResult=0, operationResult=0,
dictBuild=0, dictBuild=0,
nextArgumentIsOutFileName=0, nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0; nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0;
unsigned cLevel = 1; unsigned cLevel = 1;
unsigned cLevelLast = 1; unsigned cLevelLast = 1;
unsigned recursive = 0; unsigned recursive = 0;
@ -196,6 +215,7 @@ int main(int argCount, const char** argv)
const char* dictFileName = NULL; const char* dictFileName = NULL;
char* dynNameSpace = NULL; char* dynNameSpace = NULL;
unsigned maxDictSize = g_defaultMaxDictSize; unsigned maxDictSize = g_defaultMaxDictSize;
unsigned dictID = 0;
unsigned dictCLevel = g_defaultDictCLevel; unsigned dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel; unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
@ -205,7 +225,8 @@ int main(int argCount, const char** argv)
#endif #endif
/* init */ /* init */
(void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
(void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
filenameTable[0] = stdinmark; filenameTable[0] = stdinmark;
@ -233,13 +254,16 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; } if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
/* '-' means stdin/stdout */ /* '-' means stdin/stdout */
if (!strcmp(argument, "-")){ if (!strcmp(argument, "-")){
@ -254,12 +278,7 @@ int main(int argCount, const char** argv)
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
/* compression Level */ /* compression Level */
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
cLevel = 0; cLevel = readU32FromChar(&argument);
while ((*argument >= '0') && (*argument <= '9')) {
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
dictCLevel = cLevel; dictCLevel = cLevel;
if (dictCLevel > ZSTD_maxCLevel()) if (dictCLevel > ZSTD_maxCLevel())
CLEAN_RETURN(badusage(programName)); CLEAN_RETURN(badusage(programName));
@ -295,8 +314,11 @@ int main(int argCount, const char** argv)
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */ /* keep source file (default anyway, so useless; for gzip/xz compatibility) */
case 'k': argument++; break; case 'k': argument++; break;
/* Checksum */
case 'C': argument++; FIO_setChecksumFlag(2); break;
/* test compressed file */ /* test compressed file */
case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break; case 't': decode=1; outFileName=nulmark; argument++; break;
/* dictionary name */ /* dictionary name */
case 'o': nextArgumentIsOutFileName=1; argument++; break; case 'o': nextArgumentIsOutFileName=1; argument++; break;
@ -312,19 +334,13 @@ int main(int argCount, const char** argv)
case 'e': case 'e':
/* compression Level */ /* compression Level */
argument++; argument++;
if ((*argument>='0') && (*argument<='9')) { cLevelLast = readU32FromChar(&argument);
cLevelLast = 0;
while ((*argument >= '0') && (*argument <= '9'))
cLevelLast *= 10, cLevelLast += *argument++ - '0';
}
break; break;
/* Modify Nb Iterations (benchmark only) */ /* Modify Nb Iterations (benchmark only) */
case 'i': case 'i':
{ U32 iters= 0;
argument++; argument++;
while ((*argument >='0') && (*argument <='9')) { U32 const iters = readU32FromChar(&argument);
iters *= 10, iters += *argument++ - '0';
BMK_setNotificationLevel(displayLevel); BMK_setNotificationLevel(displayLevel);
BMK_SetNbIterations(iters); BMK_SetNbIterations(iters);
} }
@ -332,10 +348,8 @@ int main(int argCount, const char** argv)
/* cut input into blocks (benchmark only) */ /* cut input into blocks (benchmark only) */
case 'B': case 'B':
{ size_t bSize = 0;
argument++; argument++;
while ((*argument >='0') && (*argument <='9')) { size_t bSize = readU32FromChar(&argument);
bSize *= 10, bSize += *argument++ - '0';
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
if (toupper(*argument)=='M') bSize<<=20, argument++; if (toupper(*argument)=='M') bSize<<=20, argument++;
if (toupper(*argument)=='B') argument++; if (toupper(*argument)=='B') argument++;
@ -345,21 +359,17 @@ int main(int argCount, const char** argv)
break; break;
#endif /* ZSTD_NOBENCH */ #endif /* ZSTD_NOBENCH */
/* Selection level */ /* Dictionary Selection level */
case 's': argument++; case 's':
dictSelect = 0; argument++;
while ((*argument >= '0') && (*argument <= '9')) dictSelect = readU32FromChar(&argument);
dictSelect *= 10, dictSelect += *argument++ - '0';
break; break;
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++; case 'p': argument++;
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
int additionalParam = 0; BMK_setAdditionalParam(readU32FromChar(&argument));
while ((*argument >= '0') && (*argument <= '9'))
additionalParam *= 10, additionalParam += *argument++ - '0';
BMK_setAdditionalParam(additionalParam);
} else } else
#endif #endif
main_pause=1; main_pause=1;
@ -386,10 +396,15 @@ int main(int argCount, const char** argv)
if (nextArgumentIsMaxDict) { if (nextArgumentIsMaxDict) {
nextArgumentIsMaxDict = 0; nextArgumentIsMaxDict = 0;
maxDictSize = 0; maxDictSize = readU32FromChar(&argument);
while ((*argument>='0') && (*argument<='9'))
maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++;
if (toupper(*argument)=='K') maxDictSize <<= 10; if (toupper(*argument)=='K') maxDictSize <<= 10;
if (toupper(*argument)=='M') maxDictSize <<= 20;
continue;
}
if (nextArgumentIsDictID) {
nextArgumentIsDictID = 0;
dictID = readU32FromChar(&argument);
continue; continue;
} }
@ -429,6 +444,7 @@ int main(int argCount, const char** argv)
dictParams.compressionLevel = dictCLevel; dictParams.compressionLevel = dictCLevel;
dictParams.selectivityLevel = dictSelect; dictParams.selectivityLevel = dictSelect;
dictParams.notificationLevel = displayLevel; dictParams.notificationLevel = displayLevel;
dictParams.dictID = dictID;
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams); DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
#endif #endif
goto _end; goto _end;

2
projects/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
build
*Copy

View File

@ -92,80 +92,6 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|Win32" Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -242,6 +168,80 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|x64" Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -344,6 +344,10 @@
RelativePath="..\..\..\lib\common\fse_decompress.c" RelativePath="..\..\..\lib\common\fse_decompress.c"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File <File
RelativePath="..\..\..\programs\fullbench.c" RelativePath="..\..\..\programs\fullbench.c"
> >
@ -410,6 +414,10 @@
RelativePath="..\..\..\lib\common\huf_static.h" RelativePath="..\..\..\lib\common\huf_static.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\mem.h" RelativePath="..\..\..\lib\common\mem.h"
> >

View File

@ -44,7 +44,7 @@
<Tool <Tool
Name="VCCLCompilerTool" Name="VCCLCompilerTool"
Optimization="0" Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true" MinimalRebuild="true"
BasicRuntimeChecks="3" BasicRuntimeChecks="3"
@ -92,80 +92,6 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|Win32" Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -194,7 +120,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -242,6 +168,80 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|x64" Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -271,7 +271,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -332,6 +332,10 @@
RelativePath="..\..\..\programs\datagen.c" RelativePath="..\..\..\programs\datagen.c"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\entropy_common.c" RelativePath="..\..\..\lib\common\entropy_common.c"
> >
@ -357,7 +361,11 @@
> >
</File> </File>
<File <File
RelativePath="..\..\..\programs\xxhash.c" RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
> >
</File> </File>
<File <File
@ -382,6 +390,10 @@
RelativePath="..\..\..\lib\common\bitstream.h" RelativePath="..\..\..\lib\common\bitstream.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\error_private.h" RelativePath="..\..\..\lib\common\error_private.h"
> >
@ -410,10 +422,22 @@
RelativePath="..\..\..\lib\common\mem.h" RelativePath="..\..\..\lib\common\mem.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\zbuff_static.h" RelativePath="..\..\..\lib\common\zbuff_static.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\zstd.h" RelativePath="..\..\..\lib\common\zstd.h"
> >

View File

@ -44,7 +44,7 @@
<Tool <Tool
Name="VCCLCompilerTool" Name="VCCLCompilerTool"
Optimization="0" Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true" MinimalRebuild="true"
BasicRuntimeChecks="3" BasicRuntimeChecks="3"
@ -93,81 +93,6 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|Win32" Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -196,7 +121,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -245,6 +170,81 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="setargv.obj"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|x64" Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -274,7 +274,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -377,7 +377,7 @@
> >
</File> </File>
<File <File
RelativePath="..\..\..\programs\xxhash.c" RelativePath="..\..\..\lib\common\xxhash.c"
> >
</File> </File>
<File <File
@ -470,6 +470,10 @@
RelativePath="..\..\..\lib\common\mem.h" RelativePath="..\..\..\lib\common\mem.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\zbuff.h" RelativePath="..\..\..\lib\common\zbuff.h"
> >

View File

@ -44,7 +44,7 @@
<Tool <Tool
Name="VCCLCompilerTool" Name="VCCLCompilerTool"
Optimization="0" Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true" MinimalRebuild="true"
BasicRuntimeChecks="3" BasicRuntimeChecks="3"
@ -92,80 +92,6 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|Win32" Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -194,7 +120,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -242,6 +168,80 @@
Name="VCPostBuildEventTool" Name="VCPostBuildEventTool"
/> />
</Configuration> </Configuration>
<Configuration
Name="Debug|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
ConfigurationType="2"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
TargetEnvironment="3"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
UsePrecompiledHeader="0"
WarningLevel="4"
WarnAsError="true"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
TargetMachine="17"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration <Configuration
Name="Release|x64" Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)" OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
@ -271,7 +271,7 @@
Optimization="2" Optimization="2"
EnableIntrinsicFunctions="true" EnableIntrinsicFunctions="true"
OmitFramePointers="true" OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0" RuntimeLibrary="0"
EnableFunctionLevelLinking="true" EnableFunctionLevelLinking="true"
@ -352,6 +352,10 @@
RelativePath="..\..\..\lib\decompress\huf_decompress.c" RelativePath="..\..\..\lib\decompress\huf_decompress.c"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File <File
RelativePath="..\..\..\lib\compress\zbuff_compress.c" RelativePath="..\..\..\lib\compress\zbuff_compress.c"
> >
@ -418,6 +422,10 @@
RelativePath="..\..\..\lib\common\mem.h" RelativePath="..\..\..\lib\common\mem.h"
> >
</File> </File>
<File
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File <File
RelativePath="..\..\..\lib\common\zbuff.h" RelativePath="..\..\..\lib\common\zbuff.h"
> >

View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32"> <ProjectConfiguration Include="Debug|Win32">
@ -158,6 +158,7 @@
<ItemGroup> <ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" /> <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" /> <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" /> <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" /> <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" /> <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
@ -174,6 +175,7 @@
<ClInclude Include="..\..\..\lib\common\fse_static.h" /> <ClInclude Include="..\..\..\lib\common\fse_static.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" /> <ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\huf_static.h" /> <ClInclude Include="..\..\..\lib\common\huf_static.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" /> <ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" /> <ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\common\zstd.h" /> <ClInclude Include="..\..\..\lib\common\zstd.h" />

View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32"> <ProjectConfiguration Include="Debug|Win32">
@ -66,24 +66,24 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile> <ClCompile>
@ -158,30 +158,35 @@
<ItemGroup> <ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" /> <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" /> <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" /> <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" /> <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" /> <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" /> <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" /> <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" /> <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
<ClCompile Include="..\..\..\programs\datagen.c" /> <ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\fuzzer.c" /> <ClCompile Include="..\..\..\programs\fuzzer.c" />
<ClCompile Include="..\..\..\programs\xxhash.c" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="..\..\..\lib\common\fse.h" /> <ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\fse_static.h" /> <ClInclude Include="..\..\..\lib\common\fse_static.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" /> <ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\huf_static.h" /> <ClInclude Include="..\..\..\lib\common\huf_static.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" /> <ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" /> <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" /> <ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\common\zstd.h" /> <ClInclude Include="..\..\..\lib\common\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_static.h" /> <ClInclude Include="..\..\..\lib\common\zstd_static.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" /> <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" /> <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" /> <ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\xxhash.h" />
<ClInclude Include="..\..\..\programs\util.h" /> <ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32"> <ProjectConfiguration Include="Debug|Win32">
@ -20,6 +20,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" /> <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" /> <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" /> <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" /> <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
@ -41,10 +42,10 @@
<ClCompile Include="..\..\..\programs\dibio.c" /> <ClCompile Include="..\..\..\programs\dibio.c" />
<ClCompile Include="..\..\..\programs\fileio.c" /> <ClCompile Include="..\..\..\programs\fileio.c" />
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" /> <ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
<ClCompile Include="..\..\..\programs\xxhash.c" />
<ClCompile Include="..\..\..\programs\zstdcli.c" /> <ClCompile Include="..\..\..\programs\zstdcli.c" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" /> <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" /> <ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" /> <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
@ -69,7 +70,6 @@
<ClInclude Include="..\..\..\programs\dibio.h" /> <ClInclude Include="..\..\..\programs\dibio.h" />
<ClInclude Include="..\..\..\programs\fileio.h" /> <ClInclude Include="..\..\..\programs\fileio.h" />
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" /> <ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
<ClInclude Include="..\..\..\programs\xxhash.h" />
<ClInclude Include="..\..\..\programs\util.h" /> <ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup> </ItemGroup>
<PropertyGroup Label="Globals"> <PropertyGroup Label="Globals">
@ -119,27 +119,27 @@
<PropertyGroup Label="UserMacros" /> <PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<LibraryPath>$(LibraryPath)</LibraryPath> <LibraryPath>$(LibraryPath)</LibraryPath>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<LibraryPath>$(LibraryPath);</LibraryPath> <LibraryPath>$(LibraryPath);</LibraryPath>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<LibraryPath>$(LibraryPath)</LibraryPath> <LibraryPath>$(LibraryPath)</LibraryPath>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
<LibraryPath>$(LibraryPath);</LibraryPath> <LibraryPath>$(LibraryPath);</LibraryPath>
</PropertyGroup> </PropertyGroup>

View File

@ -0,0 +1,51 @@
// Microsoft Visual C++ generated resource script.
//
#include "zstd.h" /* ZSTD_VERSION_STRING */
#define APSTUDIO_READONLY_SYMBOLS
#include "verrsrc.h"
#undef APSTUDIO_READONLY_SYMBOLS
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE 9, 1
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION ZSTD_LIB_VERSION
PRODUCTVERSION ZSTD_LIB_VERSION
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_DLL
FILESUBTYPE VFT2_UNKNOWN
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904B0"
BEGIN
VALUE "CompanyName", "Yann Collet"
VALUE "FileDescription", "Fast and efficient compression algorithm"
VALUE "FileVersion", ZSTD_VERSION_STRING
VALUE "InternalName", "zstdlib.dll"
VALUE "LegalCopyright", "Copyright (C) 2013-2015, Yann Collet"
VALUE "OriginalFilename", "zstdlib.dll"
VALUE "ProductName", "Zstandard"
VALUE "ProductVersion", ZSTD_VERSION_STRING
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 1200
END
END
#endif

View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32"> <ProjectConfiguration Include="Debug|Win32">
@ -20,6 +20,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" /> <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" /> <ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" /> <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" /> <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
@ -39,6 +40,7 @@
<ClInclude Include="..\..\..\lib\common\fse_static.h" /> <ClInclude Include="..\..\..\lib\common\fse_static.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" /> <ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\huf_static.h" /> <ClInclude Include="..\..\..\lib\common\huf_static.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" /> <ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" /> <ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\common\zstd.h" /> <ClInclude Include="..\..\..\lib\common\zstd.h" />
@ -46,7 +48,9 @@
<ClInclude Include="..\..\..\lib\common\zstd_static.h" /> <ClInclude Include="..\..\..\lib\common\zstd_static.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" /> <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\programs\util.h" /> <ClInclude Include="..\..\..\programs\util.h" />
<ClInclude Include="resource.h" /> </ItemGroup>
<ItemGroup>
<ResourceCompile Include="zstdlib.rc" />
</ItemGroup> </ItemGroup>
<PropertyGroup Label="Globals"> <PropertyGroup Label="Globals">
<ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid> <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>
@ -97,28 +101,28 @@
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<TargetName>zstdlib_x86</TargetName> <TargetName>zstdlib_x86</TargetName>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental> <LinkIncremental>true</LinkIncremental>
<TargetName>zstdlib_x64</TargetName> <TargetName>zstdlib_x64</TargetName>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<TargetName>zstdlib_x86</TargetName> <TargetName>zstdlib_x86</TargetName>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental> <LinkIncremental>false</LinkIncremental>
<TargetName>zstdlib_x64</TargetName> <TargetName>zstdlib_x64</TargetName>
<IntDir>$(Platform)\$(Configuration)\</IntDir> <IntDir>$(Platform)\$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath> <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis> <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

6
projects/cmake/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
# cmake producted
CMakeCache.txt
CMakeFiles
Makefile
cmake_install.cmake
cmake_uninstall.cmake

View File

@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT)
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0) ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
ENDIF (ZSTD_LEGACY_SUPPORT) ENDIF (ZSTD_LEGACY_SUPPORT)
# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate
IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE)
ELSE ()
SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE)
ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(lib)
ADD_SUBDIRECTORY(programs) ADD_SUBDIRECTORY(programs)
@ -59,3 +52,8 @@ ADD_SUBDIRECTORY(programs)
#----------------------------------------------------------------------------- #-----------------------------------------------------------------------------
INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake) INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
ADD_EXTRA_COMPILATION_FLAGS() ADD_EXTRA_COMPILATION_FLAGS()
ADD_CUSTOM_TARGET(clean-all
COMMAND ${CMAKE_BUILD_TOOL} clean
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
)

View File

@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
SET(Sources SET(Sources
${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/entropy_common.c
${LIBRARY_DIR}/common/zstd_common.c ${LIBRARY_DIR}/common/zstd_common.c
${LIBRARY_DIR}/common/xxhash.c
${LIBRARY_DIR}/common/fse_decompress.c ${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c ${LIBRARY_DIR}/compress/fse_compress.c
${LIBRARY_DIR}/compress/huf_compress.c ${LIBRARY_DIR}/compress/huf_compress.c
@ -75,17 +76,12 @@ SET(Headers
${LIBRARY_DIR}/common/error_private.h ${LIBRARY_DIR}/common/error_private.h
${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/common/error_public.h
${LIBRARY_DIR}/common/fse.h ${LIBRARY_DIR}/common/fse.h
${LIBRARY_DIR}/common/fse_static.h
${LIBRARY_DIR}/common/huf.h ${LIBRARY_DIR}/common/huf.h
${LIBRARY_DIR}/common/huf_static.h
${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/mem.h
${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zbuff.h
${LIBRARY_DIR}/common/zbuff_static.h
${LIBRARY_DIR}/common/zstd_internal.h ${LIBRARY_DIR}/common/zstd_internal.h
${LIBRARY_DIR}/common/zstd_static.h
${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zstd.h
${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/zdict.h)
${LIBRARY_DIR}/dictBuilder/zdict_static.h)
IF (ZSTD_LEGACY_SUPPORT) IF (ZSTD_LEGACY_SUPPORT)
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@ -95,19 +91,21 @@ IF (ZSTD_LEGACY_SUPPORT)
${LIBRARY_LEGACY_DIR}/zstd_v01.c ${LIBRARY_LEGACY_DIR}/zstd_v01.c
${LIBRARY_LEGACY_DIR}/zstd_v02.c ${LIBRARY_LEGACY_DIR}/zstd_v02.c
${LIBRARY_LEGACY_DIR}/zstd_v03.c ${LIBRARY_LEGACY_DIR}/zstd_v03.c
${LIBRARY_LEGACY_DIR}/zstd_v04.c) ${LIBRARY_LEGACY_DIR}/zstd_v04.c
${LIBRARY_LEGACY_DIR}/zstd_v05.c)
SET(Headers ${Headers} SET(Headers ${Headers}
${LIBRARY_LEGACY_DIR}/zstd_legacy.h ${LIBRARY_LEGACY_DIR}/zstd_legacy.h
${LIBRARY_LEGACY_DIR}/zstd_v01.h ${LIBRARY_LEGACY_DIR}/zstd_v01.h
${LIBRARY_LEGACY_DIR}/zstd_v02.h ${LIBRARY_LEGACY_DIR}/zstd_v02.h
${LIBRARY_LEGACY_DIR}/zstd_v03.h ${LIBRARY_LEGACY_DIR}/zstd_v03.h
${LIBRARY_LEGACY_DIR}/zstd_v04.h) ${LIBRARY_LEGACY_DIR}/zstd_v04.h
${LIBRARY_LEGACY_DIR}/zstd_v05.h)
ENDIF (ZSTD_LEGACY_SUPPORT) ENDIF (ZSTD_LEGACY_SUPPORT)
IF (MSVC) IF (MSVC)
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib) SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib)
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc) SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
ENDIF (MSVC) ENDIF (MSVC)
# Split project to static and shared libraries build # Split project to static and shared libraries build
@ -120,24 +118,12 @@ IF (MSVC)
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
ENDIF (MSVC) ENDIF (MSVC)
# Define include directories
IF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common)
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common)
IF (ZSTD_LEGACY_SUPPORT)
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR})
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR})
ENDIF (ZSTD_LEGACY_SUPPORT)
ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
# Define library base name # Define library base name
IF (UNIX) IF (MSVC)
SET(LIBRARY_BASE_NAME libzstd)
ELSEIF (MSVC)
SET(LIBRARY_BASE_NAME zstdlib) SET(LIBRARY_BASE_NAME zstdlib)
ELSE () ELSE ()
MESSAGE(FATAL_ERROR "Unsupported build type") SET(LIBRARY_BASE_NAME libzstd)
ENDIF (UNIX) ENDIF (MSVC)
# Define static and shared library names # Define static and shared library names
SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}) SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME})
@ -166,12 +152,15 @@ SET_TARGET_PROPERTIES(
OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME}) OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME})
IF (UNIX) IF (UNIX)
IF ("${PREFIX}" STREQUAL "")
SET(PREFIX /usr/local) SET(PREFIX /usr/local)
ENDIF()
MESSAGE("the variable PREFIX=${PREFIX}")
SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib) SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib)
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include) SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
# install target # install target
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/zstd_buffered.h ${LIBRARY_DIR}/dictBuilder.h DESTINATION ${INSTALL_INCLUDE_DIR}) INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})

8
projects/cmake/programs/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# produced by make
datagen
fullbench
fuzzer
paramgrill
zbufftest
zstd
zstd-frugal

View File

@ -38,33 +38,23 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
SET(ROOT_DIR ../../..) SET(ROOT_DIR ../../..)
# Define programs directory, where sources and header files are located # Define programs directory, where sources and header files are located
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
SET(PROGRAMS_DIR ${ROOT_DIR}/programs) SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
IF (WORKAROUND_OUTDATED_CODE_STYLE)
# Define library directory, where sources and header files are located
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common)
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
IF (ZSTD_LEGACY_SUPPORT) IF (ZSTD_LEGACY_SUPPORT)
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR}) INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
IF (WORKAROUND_OUTDATED_CODE_STYLE)
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy)
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
ENDIF (ZSTD_LEGACY_SUPPORT) ENDIF (ZSTD_LEGACY_SUPPORT)
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY}) ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
TARGET_LINK_LIBRARIES(zstd libzstd_static) TARGET_LINK_LIBRARIES(zstd libzstd_static)
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c) ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
TARGET_LINK_LIBRARIES(fullbench libzstd_static) TARGET_LINK_LIBRARIES(fullbench libzstd_static)
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/fuzzer.c) ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c)
TARGET_LINK_LIBRARIES(fuzzer libzstd_static) TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
IF (UNIX) IF (UNIX)
@ -72,10 +62,10 @@ IF (UNIX)
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/zbufftest.c) ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c)
TARGET_LINK_LIBRARIES(zbufftest libzstd_static) TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/paramgrill.c) ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c)
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c) ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)

View File

@ -62,6 +62,7 @@ def compress_sample(tag, sample):
os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst') os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
# zstdFiles = glob.glob("*.zst*") # zstdFiles = glob.glob("*.zst*")
# print(zstdFiles) # print(zstdFiles)
print(tag + " : compression completed")
# http://stackoverflow.com/a/19711609/2132223 # http://stackoverflow.com/a/19711609/2132223

View File

@ -19,6 +19,9 @@
# Default result files # Default result files
_* _*
example
example_zstd
foo.gz
# Misc files # Misc files
*.bat *.bat

View File

@ -6,23 +6,25 @@
# Paths to static and dynamic zlib and zstd libraries # Paths to static and dynamic zlib and zstd libraries
ifneq (,$(filter Windows%,$(OS))) # Use "make ZLIBDIR=path/to/zlib" to select a path to library
STATICLIB = ../../zlib/libz.a ../lib/libzstd.a ifdef ZLIBDIR
IMPLIB = ../../zlib/libz.dll.a ../lib/libzstd.a STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a
IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a
else else
STATICLIB = -static -lz ../lib/libzstd.a STATICLIB = -static -lz ../lib/libzstd.a
IMPLIB = -lz ../lib/libzstd.a IMPLIB = -lz ../lib/libzstd.a
endif endif
ZLIBWRAPPER_PATH = . ZLIBWRAPPER_PATH = .
EXAMPLE_PATH = examples/ EXAMPLE_PATH = examples
CC = gcc CC = gcc
CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
LDFLAGS = $(LOC) LDFLAGS = $(LOC)
RM = rm -f RM = rm -f
all: example example_d all: clean test testzstd
test: example test: example
./example ./example
@ -30,6 +32,9 @@ test: example
testdll: example_d testdll: example_d
./example_d ./example_d
testzstd: example_zstd
./example_zstd
.c.o: .c.o:
$(CC) $(CFLAGS) -c -o $@ $< $(CC) $(CFLAGS) -c -o $@ $<
@ -39,16 +44,18 @@ example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB)
$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c $(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
clean: $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
-$(RM) $(EXAMPLE_PATH)/*.o
-$(RM) *.o
-$(RM) *.exe
-$(RM) foo.gz
clean:
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd
@echo Cleaning completed

View File

@ -61,12 +61,15 @@ void *myalloc(q, n, m)
void *q; void *q;
unsigned n, m; unsigned n, m;
{ {
void *buf = calloc(n, m);
q = Z_NULL; q = Z_NULL;
return calloc(n, m); /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */
return buf;
} }
void myfree(void *q, void *p) void myfree(void *q, void *p)
{ {
/* printf("myfree %p\n", p); */
q = Z_NULL; q = Z_NULL;
free(p); free(p);
} }

View File

@ -29,30 +29,35 @@
- zstd source repository : https://github.com/Cyan4973/zstd - zstd source repository : https://github.com/Cyan4973/zstd
*/ */
#include <stdio.h> /* fprintf */ #include <stdarg.h> /* va_list, for z_gzprintf */
#include <stdlib.h> /* malloc */
#include <zlib.h> #include <zlib.h>
#include "zstd_zlibwrapper.h" #include "zstd_zlibwrapper.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
#include "zstd.h" #include "zstd.h"
#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
#include "zstd_internal.h" /* MIN */
#include "zbuff.h" #include "zbuff.h"
#include "zstd_internal.h" /* defaultCustomMem */
#define Z_INFLATE_SYNC 8 #define Z_INFLATE_SYNC 8
#define ZWRAP_HEADERSIZE 4 #define ZWRAP_HEADERSIZE 4
#define ZSTD_FRAMEHEADERSIZE_MIN 5 #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
#define LOG_WRAPPER(...) // printf(__VA_ARGS__) #define LOG_WRAPPER(...) // printf(__VA_ARGS__)
#define FINISH_WITH_ERR(msg) { \ #define FINISH_WITH_GZ_ERR(msg) { \
fprintf(stderr, "ERROR: %s\n", msg); \ (void)msg; \
return Z_MEM_ERROR; \
}
#define FINISH_WITH_ERR(strm, message) { \
strm->msg = message; \
return Z_MEM_ERROR; \ return Z_MEM_ERROR; \
} }
#define FINISH_WITH_NULL_ERR(msg) { \ #define FINISH_WITH_NULL_ERR(msg) { \
fprintf(stderr, "ERROR: %s\n", msg); \ (void)msg; \
return NULL; \ return NULL; \
} }
@ -66,13 +71,29 @@ static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
void useZSTD(int turn_on) { g_useZSTD = turn_on; } void useZSTD(int turn_on) { g_useZSTD = turn_on; }
int isUsingZSTD() { return g_useZSTD; } int isUsingZSTD(void) { return g_useZSTD; }
const char * zstdVersion() { return ZSTD_VERSION_STRING; } const char * zstdVersion(void) { return ZSTD_VERSION_STRING; }
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
static void* ZWRAP_allocFunction(void* opaque, size_t size)
{
z_streamp strm = (z_streamp) opaque;
void* address = strm->zalloc(strm->opaque, 1, size);
/* printf("ZWRAP alloc %p, %d \n", address, (int)size); */
return address;
}
static void ZWRAP_freeFunction(void* opaque, void* address)
{
z_streamp strm = (z_streamp) opaque;
strm->zfree(strm->opaque, address);
/* if (address) printf("ZWRAP free %p \n", address); */
}
/* *** Compression *** */ /* *** Compression *** */
@ -80,28 +101,45 @@ typedef struct {
ZBUFF_CCtx* zbc; ZBUFF_CCtx* zbc;
size_t bytesLeft; size_t bytesLeft;
int compressionLevel; int compressionLevel;
ZSTD_customMem customMem;
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
} ZWRAP_CCtx; } ZWRAP_CCtx;
ZWRAP_CCtx* ZWRAP_createCCtx()
{
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx));
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(*zwc));
zwc->zbc = ZBUFF_createCCtx();
return zwc;
}
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
{ {
if (zwc==NULL) return 0; /* support free on NULL */ if (zwc==NULL) return 0; /* support free on NULL */
ZBUFF_freeCCtx(zwc->zbc); ZBUFF_freeCCtx(zwc->zbc);
free(zwc); zwc->customMem.customFree(zwc->customMem.opaque, zwc);
return 0; return 0;
} }
ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
{
ZWRAP_CCtx* zwc;
if (strm->zalloc && strm->zfree) {
zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx));
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
}
} else {
zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx));
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem);
if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; }
return zwc;
}
ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
const char *version, int stream_size)) const char *version, int stream_size))
{ {
@ -113,14 +151,17 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
} }
LOG_WRAPPER("- deflateInit level=%d\n", level); LOG_WRAPPER("- deflateInit level=%d\n", level);
zwc = ZWRAP_createCCtx(); zwc = ZWRAP_createCCtx(strm);
if (zwc == NULL) return Z_MEM_ERROR; if (zwc == NULL) return Z_MEM_ERROR;
if (level == Z_DEFAULT_COMPRESSION)
level = ZWRAP_DEFAULT_CLEVEL;
{ size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level); { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
zwc->compressionLevel = level; zwc->compressionLevel = level;
strm->opaque = (voidpf) zwc; strm->state = (struct internal_state*) zwc; /* use state which in not used by user */
strm->total_in = 0; strm->total_in = 0;
strm->total_out = 0; strm->total_out = 0;
return Z_OK; return Z_OK;
@ -146,7 +187,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
if (!g_useZSTD) if (!g_useZSTD)
return deflateSetDictionary(strm, dictionary, dictLength); return deflateSetDictionary(strm, dictionary, dictLength);
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type); LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type);
{ size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
@ -166,7 +207,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
return res; return res;
} }
zwc = (ZWRAP_CCtx*) strm->opaque; zwc = (ZWRAP_CCtx*) strm->state;
LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (strm->avail_in > 0) { if (strm->avail_in > 0) {
@ -183,7 +224,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
strm->avail_in -= srcSize; strm->avail_in -= srcSize;
} }
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR("Z_FULL_FLUSH is not supported!"); if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) { if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
size_t bytesLeft; size_t bytesLeft;
@ -215,7 +256,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm))
return deflateEnd(strm); return deflateEnd(strm);
} }
LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
size_t const errorCode = ZWRAP_freeCCtx(zwc); size_t const errorCode = ZWRAP_freeCCtx(zwc);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
} }
@ -253,25 +294,37 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
typedef struct { typedef struct {
ZBUFF_DCtx* zbd; ZBUFF_DCtx* zbd;
char headerBuf[ZSTD_FRAMEHEADERSIZE_MIN]; char headerBuf[ZWRAP_HEADERSIZE];
int errorCount; int errorCount;
int requiredHeaderSize;
/* zlib params */ /* zlib params */
int stream_size; int stream_size;
char *version; char *version;
int windowBits; int windowBits;
alloc_func zalloc; /* used to allocate the internal state */ ZSTD_customMem customMem;
free_func zfree; /* used to free the internal state */ z_stream allocFunc; /* copy of zalloc, zfree, opaque */
voidpf opaque; /* private data object passed to zalloc and zfree */
} ZWRAP_DCtx; } ZWRAP_DCtx;
ZWRAP_DCtx* ZWRAP_createDCtx(void) ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{ {
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx)); ZWRAP_DCtx* zwd;
if (strm->zalloc && strm->zfree) {
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL; if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(*zwd)); memset(zwd, 0, sizeof(ZWRAP_DCtx));
memcpy(&zwd->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
}
} else {
zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx));
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
return zwd; return zwd;
} }
@ -279,9 +332,9 @@ ZWRAP_DCtx* ZWRAP_createDCtx(void)
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
{ {
if (zwd==NULL) return 0; /* support free on null */ if (zwd==NULL) return 0; /* support free on null */
if (zwd->version) free(zwd->version); ZBUFF_freeDCtx(zwd->zbd);
if (zwd->zbd) ZBUFF_freeDCtx(zwd->zbd); if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
free(zwd); zwd->customMem.customFree(zwd->customMem.opaque, zwd);
return 0; return 0;
} }
@ -289,18 +342,16 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
const char *version, int stream_size)) const char *version, int stream_size))
{ {
ZWRAP_DCtx* zwd = ZWRAP_createDCtx(); ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
LOG_WRAPPER("- inflateInit\n"); LOG_WRAPPER("- inflateInit\n");
if (zwd == NULL) return Z_MEM_ERROR; if (zwd == NULL) return Z_MEM_ERROR;
zwd->stream_size = stream_size; zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
zwd->version = strdup(version); if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; }
zwd->zalloc = strm->zalloc; strcpy(zwd->version, version);
zwd->zfree = strm->zfree;
zwd->opaque = strm->opaque;
zwd->requiredHeaderSize = ZWRAP_HEADERSIZE;
strm->state = (struct internal_state*) zwd; zwd->stream_size = stream_size;
strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
strm->total_in = 0; strm->total_in = 0;
strm->total_out = 0; strm->total_out = 0;
strm->reserved = 1; /* mark as unknown steam */ strm->reserved = 1; /* mark as unknown steam */
@ -360,14 +411,14 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
size_t errorCode, dstCapacity, srcSize; size_t errorCode, dstCapacity, srcSize;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
while (strm->total_in < ZSTD_FRAMEHEADERSIZE_MIN) if (strm->total_in < ZWRAP_HEADERSIZE)
{ {
srcSize = MIN(strm->avail_in, zwd->requiredHeaderSize - strm->total_in); srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize; strm->total_in += srcSize;
strm->next_in += srcSize; strm->next_in += srcSize;
strm->avail_in -= srcSize; strm->avail_in -= srcSize;
if (strm->total_in < zwd->requiredHeaderSize) return Z_OK; if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK;
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2; z_stream strm2;
@ -396,21 +447,22 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
strm->avail_in = strm2.avail_in; strm->avail_in = strm2.avail_in;
strm->next_out = strm2.next_out; strm->next_out = strm2.next_out;
strm->avail_out = strm2.avail_out; strm->avail_out = strm2.avail_out;
strm->reserved = 0; /* mark as zlib stream */ strm->reserved = 0; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
if (flush == Z_INFLATE_SYNC) return inflateSync(strm); if (flush == Z_INFLATE_SYNC) return inflateSync(strm);
return inflate(strm, flush); return inflate(strm, flush);
} }
if (zwd->requiredHeaderSize < ZSTD_FRAMEHEADERSIZE_MIN) { zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem);
zwd->requiredHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; if (zwd->zbd == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; }
continue;
}
zwd->zbd = ZBUFF_createDCtx(); errorCode = ZBUFF_decompressInit(zwd->zbd);
{ size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
srcSize = ZSTD_FRAMEHEADERSIZE_MIN; srcSize = ZWRAP_HEADERSIZE;
dstCapacity = 0; dstCapacity = 0;
errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize); errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity); LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
@ -419,7 +471,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
return Z_MEM_ERROR; return Z_MEM_ERROR;
} }
if (strm->avail_in == 0) return Z_OK; if (strm->avail_in == 0) return Z_OK;
break;
} }
srcSize = strm->avail_in; srcSize = strm->avail_in;
@ -447,7 +498,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
{ {
int ret = Z_OK; int ret = Z_OK;
if (!strm->reserved) if (!strm->reserved)
ret = inflateEnd(strm); return inflateEnd(strm);
LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
@ -472,7 +523,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflateCopy(dest, source); return deflateCopy(dest, source);
FINISH_WITH_ERR("deflateCopy is not supported!"); FINISH_WITH_ERR(source, "deflateCopy is not supported!");
} }
@ -480,7 +531,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflateReset(strm); return deflateReset(strm);
FINISH_WITH_ERR("deflateReset is not supported!"); FINISH_WITH_ERR(strm, "deflateReset is not supported!");
} }
@ -492,18 +543,20 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); return deflateTune(strm, good_length, max_lazy, nice_length, max_chain);
FINISH_WITH_ERR("deflateTune is not supported!"); FINISH_WITH_ERR(strm, "deflateTune is not supported!");
} }
#if ZLIB_VERNUM >= 0x1260
ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm,
unsigned *pending, unsigned *pending,
int *bits)) int *bits))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflatePending(strm, pending, bits); return deflatePending(strm, pending, bits);
FINISH_WITH_ERR("deflatePending is not supported!"); FINISH_WITH_ERR(strm, "deflatePending is not supported!");
} }
#endif
ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
@ -512,7 +565,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflatePrime(strm, bits, value); return deflatePrime(strm, bits, value);
FINISH_WITH_ERR("deflatePrime is not supported!"); FINISH_WITH_ERR(strm, "deflatePrime is not supported!");
} }
@ -521,22 +574,23 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return deflateSetHeader(strm, head); return deflateSetHeader(strm, head);
FINISH_WITH_ERR("deflateSetHeader is not supported!"); FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!");
} }
/* Advanced compression functions */ /* Advanced compression functions */
#if ZLIB_VERNUM >= 0x1280
ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm,
Bytef *dictionary, Bytef *dictionary,
uInt *dictLength)) uInt *dictLength))
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateGetDictionary(strm, dictionary, dictLength); return inflateGetDictionary(strm, dictionary, dictLength);
FINISH_WITH_ERR("inflateGetDictionary is not supported!"); FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!");
} }
#endif
ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
@ -544,7 +598,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return inflateCopy(dest, source); return inflateCopy(dest, source);
FINISH_WITH_ERR("inflateCopy is not supported!"); FINISH_WITH_ERR(source, "inflateCopy is not supported!");
} }
@ -552,17 +606,29 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateReset(strm); return inflateReset(strm);
FINISH_WITH_ERR("inflateReset is not supported!"); FINISH_WITH_ERR(strm, "inflateReset is not supported!");
} }
#if ZLIB_VERNUM >= 0x1240
ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
int windowBits)) int windowBits))
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateReset2(strm, windowBits); return inflateReset2(strm, windowBits);
FINISH_WITH_ERR("inflateReset2 is not supported!"); FINISH_WITH_ERR(strm, "inflateReset2 is not supported!");
} }
#endif
#if ZLIB_VERNUM >= 0x1240
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
{
if (!strm->reserved)
return inflateMark(strm);
FINISH_WITH_ERR(strm, "inflateMark is not supported!");
}
#endif
ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
@ -571,15 +637,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
{ {
if (!strm->reserved) if (!strm->reserved)
return inflatePrime(strm, bits, value); return inflatePrime(strm, bits, value);
FINISH_WITH_ERR("inflatePrime is not supported!"); FINISH_WITH_ERR(strm, "inflatePrime is not supported!");
}
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
{
if (!strm->reserved)
return inflateMark(strm);
FINISH_WITH_ERR("inflateMark is not supported!");
} }
@ -588,7 +646,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm,
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateGetHeader(strm, head); return inflateGetHeader(strm, head);
FINISH_WITH_ERR("inflateGetHeader is not supported!"); FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!");
} }
@ -599,7 +657,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits,
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateBackInit_(strm, windowBits, window, version, stream_size); return inflateBackInit_(strm, windowBits, window, version, stream_size);
FINISH_WITH_ERR("inflateBackInit is not supported!"); FINISH_WITH_ERR(strm, "inflateBackInit is not supported!");
} }
@ -609,7 +667,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm,
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateBack(strm, in, in_desc, out, out_desc); return inflateBack(strm, in, in_desc, out, out_desc);
FINISH_WITH_ERR("inflateBack is not supported!"); FINISH_WITH_ERR(strm, "inflateBack is not supported!");
} }
@ -617,7 +675,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
{ {
if (!strm->reserved) if (!strm->reserved)
return inflateBackEnd(strm); return inflateBackEnd(strm);
FINISH_WITH_ERR("inflateBackEnd is not supported!"); FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!");
} }
@ -634,9 +692,9 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
if (!g_useZSTD) if (!g_useZSTD)
return compress(dest, destLen, source, sourceLen); return compress(dest, destLen, source, sourceLen);
size_t dstCapacity = *destLen; { size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
*destLen = errorCode; *destLen = errorCode;
} }
@ -651,8 +709,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen,
if (!g_useZSTD) if (!g_useZSTD)
return compress2(dest, destLen, source, sourceLen, level); return compress2(dest, destLen, source, sourceLen, level);
size_t dstCapacity = *destLen; { size_t dstCapacity = *destLen;
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
*destLen = errorCode; *destLen = errorCode;
} }
@ -676,8 +734,8 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen,
// if (!g_useZSTD) // if (!g_useZSTD)
return uncompress(dest, destLen, source, sourceLen); return uncompress(dest, destLen, source, sourceLen);
size_t dstCapacity = *destLen; { size_t dstCapacity = *destLen;
{ size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
*destLen = errorCode; *destLen = errorCode;
} }
@ -703,19 +761,45 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode))
} }
#if ZLIB_VERNUM >= 0x1240
ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzbuffer(file, size); return gzbuffer(file, size);
FINISH_WITH_ERR("gzbuffer is not supported!"); FINISH_WITH_GZ_ERR("gzbuffer is not supported!");
} }
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
{
if (!g_useZSTD)
return gzoffset(file);
FINISH_WITH_GZ_ERR("gzoffset is not supported!");
}
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
{
if (!g_useZSTD)
return gzclose_r(file);
FINISH_WITH_GZ_ERR("gzclose_r is not supported!");
}
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
{
if (!g_useZSTD)
return gzclose_w(file);
FINISH_WITH_GZ_ERR("gzclose_w is not supported!");
}
#endif
ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzsetparams(file, level, strategy); return gzsetparams(file, level, strategy);
FINISH_WITH_ERR("gzsetparams is not supported!"); FINISH_WITH_GZ_ERR("gzsetparams is not supported!");
} }
@ -723,7 +807,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzread(file, buf, len); return gzread(file, buf, len);
FINISH_WITH_ERR("gzread is not supported!"); FINISH_WITH_GZ_ERR("gzread is not supported!");
} }
@ -732,11 +816,15 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file,
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzwrite(file, buf, len); return gzwrite(file, buf, len);
FINISH_WITH_ERR("gzwrite is not supported!"); FINISH_WITH_GZ_ERR("gzwrite is not supported!");
} }
#if ZLIB_VERNUM >= 0x1260
ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...)) ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
#else
ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...))
#endif
{ {
if (!g_useZSTD) { if (!g_useZSTD) {
int ret; int ret;
@ -750,7 +838,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
// printf("gzprintf ret=%d\n", ret); // printf("gzprintf ret=%d\n", ret);
return ret; return ret;
} }
FINISH_WITH_ERR("gzprintf is not supported!"); FINISH_WITH_GZ_ERR("gzprintf is not supported!");
} }
@ -758,7 +846,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzputs(file, s); return gzputs(file, s);
FINISH_WITH_ERR("gzputs is not supported!"); FINISH_WITH_GZ_ERR("gzputs is not supported!");
} }
@ -774,15 +862,19 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzputc(file, c); return gzputc(file, c);
FINISH_WITH_ERR("gzputc is not supported!"); FINISH_WITH_GZ_ERR("gzputc is not supported!");
} }
#if ZLIB_VERNUM == 0x1260
ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file))
#else
ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file))
#endif
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzgetc(file); return gzgetc(file);
FINISH_WITH_ERR("gzgetc is not supported!"); FINISH_WITH_GZ_ERR("gzgetc is not supported!");
} }
@ -790,7 +882,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzungetc(c, file); return gzungetc(c, file);
FINISH_WITH_ERR("gzungetc is not supported!"); FINISH_WITH_GZ_ERR("gzungetc is not supported!");
} }
@ -798,7 +890,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzflush(file, flush); return gzflush(file, flush);
FINISH_WITH_ERR("gzflush is not supported!"); FINISH_WITH_GZ_ERR("gzflush is not supported!");
} }
@ -806,7 +898,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzseek(file, offset, whence); return gzseek(file, offset, whence);
FINISH_WITH_ERR("gzseek is not supported!"); FINISH_WITH_GZ_ERR("gzseek is not supported!");
} }
@ -814,7 +906,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzrewind(file); return gzrewind(file);
FINISH_WITH_ERR("gzrewind is not supported!"); FINISH_WITH_GZ_ERR("gzrewind is not supported!");
} }
@ -822,15 +914,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gztell(file); return gztell(file);
FINISH_WITH_ERR("gztell is not supported!"); FINISH_WITH_GZ_ERR("gztell is not supported!");
}
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
{
if (!g_useZSTD)
return gzoffset(file);
FINISH_WITH_ERR("gzoffset is not supported!");
} }
@ -838,7 +922,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzeof(file); return gzeof(file);
FINISH_WITH_ERR("gzeof is not supported!"); FINISH_WITH_GZ_ERR("gzeof is not supported!");
} }
@ -846,7 +930,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzdirect(file); return gzdirect(file);
FINISH_WITH_ERR("gzdirect is not supported!"); FINISH_WITH_GZ_ERR("gzdirect is not supported!");
} }
@ -854,23 +938,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file))
{ {
if (!g_useZSTD) if (!g_useZSTD)
return gzclose(file); return gzclose(file);
FINISH_WITH_ERR("gzclose is not supported!"); FINISH_WITH_GZ_ERR("gzclose is not supported!");
}
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
{
if (!g_useZSTD)
return gzclose_r(file);
FINISH_WITH_ERR("gzclose_r is not supported!");
}
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
{
if (!g_useZSTD)
return gzclose_w(file);
FINISH_WITH_ERR("gzclose_w is not supported!");
} }

View File

@ -40,10 +40,17 @@ extern "C" {
#define Z_PREFIX #define Z_PREFIX
#include <zlib.h> #include <zlib.h>
#if !defined(z_const)
#if ZLIB_VERNUM >= 0x1260
#define z_const const
#else
#define z_const
#endif
#endif
void useZSTD(int turn_on); void useZSTD(int turn_on);
int isUsingZSTD(); int isUsingZSTD(void);
const char * zstdVersion(); const char * zstdVersion(void);
#if defined (__cplusplus) #if defined (__cplusplus)