facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #725 from facebook/advancedAPI2 

New Advanced API
dev
Yann Collet 2017-06-23 09:50:47 -07:00 committed by GitHub
parent 0d0fea96bc 6122af9cf5
commit ef269c1b68
36 changed files with 3777 additions and 1798 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -108,7 +108,7 @@ clean:
#------------------------------------------------------------------------------
# make install is validated only for Linux, OSX, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT))
HOST_OS = POSIX
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
@ -227,7 +227,7 @@ msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason
msan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) $*
asan32: clean
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
@ -239,7 +239,8 @@ uasan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
tsan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
apt-install:
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)

12
NEWS
View File

@ -1,3 +1,15 @@
v1.3.0
cli : new : `--list` command, by Paul Cruz
cli : changed : `-t *` continue processing list after a decompression error
API : added : ZSTD_versionString()
API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
API exp : clarified presentation of memory estimation / measurement functions.
new : contrib/seekable_format, demo and API, by Sean Purcell
changed : contrib/linux-kernel, updated version and license, by Nick Terrell
v1.2.0
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell

62
build/VS2008/fullbench/fullbench.vcproj
View File

@ -328,10 +328,6 @@
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath="..\..\..\programs\datagen.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
@ -340,10 +336,6 @@
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\fse_decompress.c"
>
@ -352,36 +344,60 @@
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\tests\fullbench.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\pool.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\threading.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\huf_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\programs\datagen.c"
>
</File>
<File
RelativePath="..\..\..\tests\fullbench.c"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath="..\..\..\lib\zstd.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\bitstream.h"
>
@ -419,11 +435,11 @@
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
RelativePath="..\..\..\lib\common\zstd_internal.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_internal.h"
RelativePath="..\..\..\lib\common\zstd_static.h"
>
</File>
<File
@ -431,7 +447,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_static.h"
RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
>
</File>
</Filter>

12
build/VS2010/fullbench/fullbench.vcxproj
View File

@ -156,26 +156,32 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\error_private.c" />
<ClCompile Include="..\..\..\lib\common\pool.c" />
<ClCompile Include="..\..\..\lib\common\threading.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\pool.h" />
<ClInclude Include="..\..\..\lib\common\threading.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\util.h" />

406
doc/zstd_manual.html
View File

@ -13,7 +13,7 @@
<li><a href="#Chapter3">Simple API</a></li>
<li><a href="#Chapter4">Explicit memory management</a></li>
<li><a href="#Chapter5">Simple dictionary API</a></li>
<li><a href="#Chapter6">Fast dictionary API</a></li>
<li><a href="#Chapter6">Bulk processing dictionary API</a></li>
<li><a href="#Chapter7">Streaming</a></li>
<li><a href="#Chapter8">Streaming compression - HowTo</a></li>
<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
@ -31,26 +31,27 @@
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
zstd, short for Zstandard, is a fast lossless compression algorithm,
targeting real-time compression scenarios at zlib-level and better compression ratios.
The zstd compression library provides in-memory compression and decompression functions.
The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
Compression can be done in:
- a single step (described as Simple API)
- a single step, reusing a context (described as Explicit memory management)
- unbounded multiple steps (described as Streaming compression)
The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
The compression ratio achievable on small data can be highly improved using a dictionary in:
- a single step (described as Simple dictionary API)
- a single step, reusing a dictionary (described as Fast dictionary API)
Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
These APIs shall never be used with a dynamic library.
Advanced experimental APIs shall never be used with a dynamic library.
They are not "stable", their definition may change in the future. Only static linking is allowed.
<BR></pre>
<a name="Chapter2"></a><h2>Version</h2><pre></pre>
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< library version number; to be used when checking dll version */<b>
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< useful to check dll version */<b>
</b></pre><BR>
<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
@ -66,28 +67,24 @@
<pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
`dstCapacity` is an upper bound of originalSize.
`dstCapacity` is an upper bound of originalSize to regenerate.
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()).
</p></pre><BR>
<pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
</b><p> NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
frame, but distinguishes empty frames from frames with an unknown size, or errors.
Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
concatenated frames in one buffer, and so is more general.
As a result however, it requires more computation and entire frames to be passed to it,
as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
</b><p> NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize().
ZSTD_getFrameContentSize() works the same way,
returning the decompressed size of a single frame,
but distinguishes empty frames from frames with an unknown size, or errors.
'src' is the start of a zstd compressed frame.
@return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode.
When `return==0`, data to decompress could be any size.
In which case, it's necessary to use streaming mode to decompress data.
Optionally, application can still use ZSTD_decompress() while relying on implied limits.
Optionally, application can use ZSTD_decompress() while relying on implied limits.
(For example, data may be necessarily cut into blocks <= 16 KB).
note 2 : decompressed size is always present when compression is done with ZSTD_compress()
note 3 : decompressed size can be very large (64-bits value),
@ -96,7 +93,7 @@
note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
Always ensure result fits within application's authorized limits.
Each application can set its own limits.
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more.
note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameHeader() to know more.
</p></pre><BR>
<h3>Helper functions</h3><pre></pre><b><pre>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
@ -114,20 +111,26 @@ const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable strin
ZSTD_CCtx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
</pre></b><BR>
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()).
</p></pre><BR>
<h3>Decompression context</h3><pre> When decompressing many times,
it is recommended to allocate a context just once, and re-use it for each successive compression operation.
it is recommended to allocate a context only once,
and re-use it for each successive compression operation.
This will make workload friendlier for system's memory.
Use one context per thread for parallel execution in multi-threaded environments.
Use one context per thread for parallel execution.
</pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</pre></b><BR>
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()).
<pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx())
</p></pre><BR>
<a name="Chapter5"></a><h2>Simple dictionary API</h2><pre></pre>
@ -137,32 +140,33 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
int compressionLevel);
</b><p> Compression using a predefined Dictionary (see dictBuilder/zdict.h).
Note : This function loads the dictionary, resulting in significant startup delay.
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</b><p> Compression using a predefined Dictionary (see dictBuilder/zdict.h).
Note : This function loads the dictionary, resulting in significant startup delay.
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</p></pre><BR>
<pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
</b><p> Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
Dictionary must be identical to the one used during compression.
Note : This function loads the dictionary, resulting in significant startup delay.
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</b><p> Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
Dictionary must be identical to the one used during compression.
Note : This function loads the dictionary, resulting in significant startup delay.
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</p></pre><BR>
<a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
<a name="Chapter6"></a><h2>Bulk processing dictionary API</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
`dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
</b><p> When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
`dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict
</p></pre><BR>
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
</b><p> Function frees memory allocated by ZSTD_createCDict().
</b><p> Function frees memory allocated by ZSTD_createCDict().
</p></pre><BR>
<pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@ -176,20 +180,20 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
dictBuffer can be released after DDict creation, as its content is copied inside DDict
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
dictBuffer can be released after DDict creation, as its content is copied inside DDict
</p></pre><BR>
<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
</b><p> Function frees memory allocated with ZSTD_createDDict()
</b><p> Function frees memory allocated with ZSTD_createDDict()
</p></pre><BR>
<pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict);
</b><p> Decompression using a digested Dictionary.
Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times.
</b><p> Decompression using a digested Dictionary.
Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times.
</p></pre><BR>
<a name="Chapter7"></a><h2>Streaming</h2><pre></pre>
@ -236,15 +240,17 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_endStream() instructs to finish a frame.
It will perform a flush and write frame epilogue.
The epilogue is required for decoders to consider a frame completed.
Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
In which case, call again ZSTD_endStream() to complete the flush.
@return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
@return : 0 if frame fully completed and fully flushed,
or >0 if some data is still present within internal buffer
(value is minimum size estimation for remaining data to flush, but it could be more)
or an error code, which can be tested using ZSTD_isError().
<BR></pre>
<pre><b>typedef ZSTD_CCtx ZSTD_CStream; </b>/**< CCtx and CStream are effectively same object */<b>
<pre><b>typedef ZSTD_CCtx ZSTD_CStream; </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b>
</b></pre><BR>
<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@ -279,6 +285,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
<BR></pre>
<pre><b>typedef ZSTD_DCtx ZSTD_DStream; </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b>
</b></pre><BR>
<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
</pre></b><BR>
@ -298,7 +306,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
<a name="Chapter11"></a><h2>Advanced types</h2><pre></pre>
<pre><b>typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; </b>/* from faster to stronger */<b>
<pre><b>typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; </b>/* from faster to stronger */<b>
</b></pre><BR>
<pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
@ -331,6 +340,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
</b>/* use this constant to defer to stdlib's functions */<b>
static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
</pre></b><BR>
<a name="Chapter12"></a><h2>Frame size functions</h2><pre></pre>
@ -395,19 +406,20 @@ size_t ZSTD_estimateDCtxSize(void);
</b><p> These functions make it possible to estimate memory usage
of a future target object, before its allocation,
given a set of parameters, which vary depending on target object.
The objective is to guide decision before allocation.
The objective is to guide decision before allocation.
Note : CCtx estimation is only correct for single-threaded compression
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
</b><p> Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
an internal ?Dict will be created, which size is not estimated.
In this case, get additional size by using ZSTD_estimate?DictSize
an internal ?Dict will be created, which size is not estimated here.
In this case, get total size by adding ZSTD_estimate?DictSize
</p></pre><BR>
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Note : if dictionary is created "byReference", reduce estimation by dictSize
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
</b><p> Note : dictionary created "byReference" are smaller
</p></pre><BR>
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
@ -416,6 +428,23 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Create a ZSTD compression context using external alloc and free functions
</p></pre><BR>
<pre><b>ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
</b><p> workspace: The memory area to emplace the context into.
Provided pointer must 8-bytes aligned.
It must outlive context usage.
workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
to determine how large workspace must be to support scenario.
@return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
Note : zstd will never resize nor malloc() when using a static cctx.
If it needs more memory than available, it will simply error out.
Note 2 : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally too.
Limitation 1 : currently not compatible with internal CDict creation, such as
ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
Limitation 2 : currently not compatible with multi-threading
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_p_forceWindow, </b>/* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */<b>
ZSTD_p_forceRawDict </b>/* Force loading dictionary in "content-only" mode (no header analysis) */<b>
@ -432,11 +461,33 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict
</p></pre><BR>
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
<pre><b>typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
</b></pre><BR>
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
</p></pre><BR>
<pre><b>ZSTD_CDict* ZSTD_initStaticCDict(
void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams);
</b><p> Generate a digested dictionary in provided memory area.
workspace: The memory area to emplace the dictionary into.
Provided pointer must 8-bytes aligned.
It must outlive dictionary usage.
workspaceSize: Use ZSTD_estimateCDictSize()
to determine how large workspace must be.
cParams : use ZSTD_getCParams() to transform a compression level
into its relevants cParams.
@return : pointer to ZSTD_CDict*, or NULL if error (size too small)
Note : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally.
</p></pre><BR>
<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
</b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
`estimatedSrcSize` value is optional, select 0 if not known
@ -452,8 +503,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
</p></pre><BR>
<pre><b>ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
</b><p> optimize params for a given `srcSize` and `dictSize`.
both values are optional, select `0` if unknown.
</b><p> optimize params for a given `srcSize` and `dictSize`.
both values are optional, select `0` if unknown.
</p></pre><BR>
<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
@ -484,10 +535,28 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Create a ZSTD decompression context using external alloc and free functions
</p></pre><BR>
<pre><b>ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
</b><p> workspace: The memory area to emplace the context into.
Provided pointer must 8-bytes aligned.
It must outlive context usage.
workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
to determine how large workspace must be to support scenario.
@return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
Note : zstd will never resize nor malloc() when using a static dctx.
If it needs more memory than available, it will simply error out.
Note 2 : static dctx is incompatible with legacy support
Note 3 : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally.
Limitation : currently not compatible with internal DDict creation,
such as ZSTD_initDStream_usingDict().
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
Dictionary content is simply referenced, and therefore stays in dictBuffer.
It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict
Dictionary content is referenced, and therefore stays in dictBuffer.
It is important that dictBuffer outlives DDict,
it must remain read accessible throughout the lifetime of DDict
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
@ -495,6 +564,21 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference);
</b><p> Generate a digested dictionary in provided memory area.
workspace: The memory area to emplace the dictionary into.
Provided pointer must 8-bytes aligned.
It must outlive dictionary usage.
workspaceSize: Use ZSTD_estimateDDictSize()
to determine how large workspace must be.
@return : pointer to ZSTD_DDict*, or NULL if error (size too small)
Note : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally.
</p></pre><BR>
<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
</b><p> Provides the dictID stored within dictionary.
if @return == 0, the dictionary is not conformant with Zstandard specification.
@ -516,18 +600,18 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
Note : this use case also happens when using a non-conformant dictionary.
- `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
- This is not a Zstandard frame.
When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code.
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
</p></pre><BR>
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */<b>
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
</pre></b><BR>
<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
</b><p> start a new compression job, using same parameters from previous job.
@ -593,7 +677,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
and the dictionary ID used.
@ -607,7 +691,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
Start decompression, with ZSTD_decompressBegin().
If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
@ -639,7 +724,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_decompressContinue() always returns 0.
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
For purposes of decompression, it is valid in both cases to skip the frame using
ZSTD_findFrameCompressedSize to find its size in bytes.
@ -649,12 +734,193 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
</pre></b><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR>
<h3>New advanced API (experimental, and compression only)</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>typedef enum {
</b>/* compression parameters */<b>
ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means "do not change cLevel". */
ZSTD_p_windowLog, </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b>
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "do not change windowLog". */
ZSTD_p_hashLog, </b>/* Size of the probe table, as a power of 2.<b>
* Resulting table size is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
* Larger tables improve compression ratio of strategies <= dFast,
* and improve speed of strategies > dFast.
* Special: value 0 means "do not change hashLog". */
ZSTD_p_chainLog, </b>/* Size of the full-search table, as a power of 2.<b>
* Resulting table size is (1 << (chainLog+2)).
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* Special: value 0 means "do not change chainLog". */
ZSTD_p_searchLog, </b>/* Number of search attempts, as a power of 2.<b>
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* Special: value 0 means "do not change searchLog". */
ZSTD_p_minMatch, </b>/* Minimum size of searched matches (note : repCode matches can be smaller).<b>
* Larger values make faster compression and decompression, but decrease ratio.
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
* Note that currently, for all strategies < btopt, effective minimum is 4.
* Note that currently, for all strategies > fast, effective maximum is 6.
* Special: value 0 means "do not change minMatchLength". */
ZSTD_p_targetLength, </b>/* Only useful for strategies >= btopt.<b>
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger and slower.
* Special: value 0 means "do not change targetLength". */
ZSTD_p_compressionStrategy, </b>/* See ZSTD_strategy enum definition.<b>
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
#if 0
ZSTD_p_windowSize, </b>/* Maximum allowed back-reference distance.<b>
* Can be set to a more precise value than windowLog.
* Will be transparently reduced to closest possible inferior value
* (see Zstandard compression format) */
</b>/* Not ready yet ! */<b>
#endif
</b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1) */<b>
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
</b>/* dictionary parameters */<b>
ZSTD_p_refDictContent=300, </b>/* Content of dictionary content will be referenced, instead of copied (default:0).<b>
* This avoids duplicating dictionary content.
* But it also requires that dictionary buffer outlives its users */
</b>/* Not ready yet ! <=================================== */<b>
ZSTD_p_dictMode, </b>/* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
</b>/* multi-threading parameters */<b>
ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b>
* More threads improve speed, but also increase memory usage.
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
* Special: value 0 means "do not change nbThreads" */
ZSTD_p_jobSize, </b>/* Size of a compression job. Each compression job is completed in parallel.<b>
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog, </b>/* Size of previous input reloaded at the beginning of each job.<b>
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
</b>/* advanced parameters - may not remain available after API update */<b>
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-references to remain < windowSize,<b>
* even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
</b><p> Set one compression parameter, selected by enum ZSTD_cParameter.
Note : when `value` is an enum, cast it to unsigned for proper type checking.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
</b><p> Total input data size to be compressed as a single frame.
This value will be controlled at the end, and result in error if not respected.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : 0 means zero, empty.
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
Note 2 : If all data is provided and consumed in a single round,
this value is overriden by srcSize instead.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
</b><p> Create an internal CDict from dict buffer.
Decompression will have to use same buffer.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
meaning "return to no-dictionary mode".
Note 1 : Dictionary content will be copied internally,
except if ZSTD_p_refDictContent is set.
Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
For this reason, compression parameters cannot be changed anymore after loading a dictionary.
It's also a CPU-heavy operation, with non-negligible impact on latency.
Note 3 : Dictionary will be used for all future compression jobs.
To return to "no-dictionary" situation, load a NULL dictionary
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
</b><p> Ref a prepared dictionary, to be used for all next compression jobs.
Note that compression parameters are enforced from within CDict,
and supercede any compression parameter previously set within CCtx.
The dictionary will remain valid for future compression jobs using same CCtx.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : adding a NULL CDict means "return to no-dictionary mode".
Note 1 : Currently, only one dictionary can be managed.
Adding a new dictionary effectively "discards" any previous one.
Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); </b>/* Not ready yet ! <===================================== */<b>
</b><p> Reference a prefix (content-only dictionary) to bootstrap next compression job.
Decompression will have to use same prefix.
Prefix is only used once. Tables are discarded at end of compression job.
If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
Note 1 : Prefix buffer is referenced. It must outlive compression job.
Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU-heavy operation, with non-negligible impact on latency.
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_e_continue=0, </b>/* collect more data, encoder transparently decides when to output result, for optimal conditions */<b>
ZSTD_e_flush, </b>/* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */<b>
ZSTD_e_end </b>/* flush any remaining data and ends current frame. Any future compression starts a new frame. */<b>
} ZSTD_EndDirective;
</b></pre><BR>
<pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
</b><p> Behave about the same as ZSTD_compressStream. To note :
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
- Compression parameters cannot be changed once compression is started.
- *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
- *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
- @return provides the minimum amount of data still to flush from internal buffers
or an error code, which can be tested using ZSTD_isError().
if @return != 0, flush is not fully completed, there is some data left within internal buffers.
- after a ZSTD_e_end directive, if internal buffer is not fully flushed,
only ZSTD_e_end or ZSTD_e_flush operations are allowed.
It is necessary to fully flush internal buffers
before starting a new compression job, or changing compression parameters.
</p></pre><BR>
<pre><b>void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); </b>/* Not ready yet ! */<b>
</b><p> Return a CCtx to clean state.
Useful after an error, or to interrupt an ongoing compression job and start a new one.
Any internal data not yet flushed is cancelled.
Dictionary (if any) is dropped.
It's possible to modify compression parameters after a reset.
</p></pre><BR>
<pre><b>size_t ZSTD_compress_generic_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos,
ZSTD_EndDirective endOp);
</b><p> Same as ZSTD_compress_generic(),
but using only integral types as arguments.
Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
but can be helpful for binders from dynamic languages
which have troubles handling structures containing memory pointers.
</p></pre><BR>
<a name="Chapter20"></a><h2>Block functions</h2><pre>
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
@ -667,7 +933,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ copyCCtx() and copyDCtx() can be used too
- Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@ -680,7 +946,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
Use ZSTD_insertBlock() for such a case.
<BR></pre>
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>

15
lib/common/bitstream.h
View File

@ -58,7 +58,9 @@ extern "C" {
#if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
# include <assert.h>
#else
# define assert(condition) ((void)0)
# ifndef assert
# define assert(condition) ((void)0)
# endif
#endif
@ -73,6 +75,7 @@ extern "C" {
#define STREAM_ACCUMULATOR_MIN_64 57
#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
@ -307,19 +310,19 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
/* fall-through */
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
/* fall-through */
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
/* fall-through */
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
/* fall-through */
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
/* fall-through */
default: break;
}
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];

3
lib/common/error_private.c
View File

@ -24,7 +24,8 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is not supported";
case PREFIX(compressionParameter_outOfBound): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";

14
lib/common/mem.h
View File

@ -352,20 +352,6 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
#endif

12
lib/common/pool.c
View File

@ -146,6 +146,13 @@ void POOL_free(POOL_ctx *ctx) {
free(ctx);
}
size_t POOL_sizeof(POOL_ctx *ctx) {
if (ctx==NULL) return 0; /* supports sizeof NULL */
return sizeof(*ctx)
+ ctx->queueSize * sizeof(POOL_job)
+ ctx->numThreads * sizeof(pthread_t);
}
void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
if (!ctx) { return; }
@ -191,4 +198,9 @@ void POOL_add(void *ctx, POOL_function function, void *opaque) {
function(opaque);
}
size_t POOL_sizeof(POOL_ctx *ctx) {
if (ctx==NULL) return 0; /* supports sizeof NULL */
return sizeof(*ctx);
}
#endif /* ZSTD_MULTITHREAD */

5
lib/common/pool.h
View File

@ -32,6 +32,11 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
*/
void POOL_free(POOL_ctx *ctx);
/*! POOL_sizeof() :
return memory usage of pool returned by POOL_create().
*/
size_t POOL_sizeof(POOL_ctx *ctx);
/*! POOL_function :
The function type that can be added to a thread pool.
*/

43
lib/common/zstd_common.c
View File

@ -12,7 +12,8 @@
/*-*************************************
* Dependencies
***************************************/
#include <stdlib.h> /* malloc */
#include <stdlib.h> /* malloc, calloc, free */
#include <string.h> /* memset */
#include "error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
@ -21,7 +22,7 @@
/*-****************************************
* Version
******************************************/
unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
@ -49,27 +50,31 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString
/*=**************************************************************
* Custom allocator
****************************************************************/
/* default uses stdlib */
void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
return address;
}
void ZSTD_defaultFreeFunction(void* opaque, void* address)
{
(void)opaque;
free(address);
}
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
{
return customMem.customAlloc(customMem.opaque, size);
if (customMem.customAlloc)
return customMem.customAlloc(customMem.opaque, size);
return malloc(size);
}
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc) {
/* calloc implemented as malloc+memset;
* not as efficient as calloc, but next best guess for custom malloc */
void* const ptr = customMem.customAlloc(customMem.opaque, size);
memset(ptr, 0, size);
return ptr;
}
return calloc(1, size);
}
void ZSTD_free(void* ptr, ZSTD_customMem customMem)
{
if (ptr!=NULL)
customMem.customFree(customMem.opaque, ptr);
if (ptr!=NULL) {
if (customMem.customFree)
customMem.customFree(customMem.opaque, ptr);
else
free(ptr);
}
}

10
lib/common/zstd_errors.h
View File

@ -35,8 +35,11 @@ extern "C" {
#endif
/*-****************************************
* error codes list
******************************************/
* error codes list
* note : this API is still considered unstable
* it should not be used with a dynamic library
* only static linking is allowed
******************************************/
typedef enum {
ZSTD_error_no_error,
ZSTD_error_GENERIC,
@ -47,6 +50,7 @@ typedef enum {
ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_compressionParameter_outOfBound,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
ZSTD_error_stage_wrong,
@ -67,7 +71,7 @@ typedef enum {
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
which can be used to compare with enum list published above */
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);

69
lib/common/zstd_internal.h
View File

@ -18,6 +18,7 @@
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
@ -50,9 +51,36 @@
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
/*-*************************************
* Debug
***************************************/
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
# include <assert.h>
#else
# ifndef assert
# define assert(condition) ((void)0)
# endif
#endif
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h>
static unsigned g_debugLevel = ZSTD_DEBUG;
# define DEBUGLOG(l, ...) { \
if (l<=g_debugLevel) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else
# define DEBUGLOG(l, ...) {} /* disabled */
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
/*-*************************************
@ -235,15 +263,10 @@ typedef struct {
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
/* custom memory allocation functions */
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
void ZSTD_defaultFreeFunction(void* opaque, void* address);
#ifndef ZSTD_DLL_IMPORT
static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
#endif
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
void ZSTD_free(void* ptr, ZSTD_customMem customMem);
@ -281,4 +304,34 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
#if 0
/*! ZSTD_compressBegin_internal() :
* innermost initialization function. Private use only.
* expects params to be valid.
* must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */
size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff);
#endif
/*! ZSTD_initCStream_internal() :
* Private use only. Init streaming operation.
* expects params to be valid.
* must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
/*! ZSTD_getParamsFromCDict() :
* as the name implies */
ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict);
#endif /* ZSTD_CCOMMON_H_MODULE */

1338
lib/compress/zstd_compress.c
View File

File diff suppressed because it is too large Load Diff

45
lib/compress/zstd_opt.h
View File

@ -43,6 +43,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
if (ssPtr->litLengthSum == 0) {
if (srcSize <= 1024) ssPtr->staticPrices = 1;
assert(ssPtr->litFreq!=NULL);
for (u=0; u<=MaxLit; u++)
ssPtr->litFreq[u] = 0;
for (u=0; u<srcSize; u++)
@ -201,6 +202,20 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
}
/* function safe only for comparisons */
MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
/* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
@ -234,12 +249,12 @@ static U32 ZSTD_insertBtAndGetAllMatches (
{
const BYTE* const base = zc->base;
const U32 current = (U32)(ip-base);
const U32 hashLog = zc->params.cParams.hashLog;
const U32 hashLog = zc->appliedParams.cParams.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const hashTable = zc->hashTable;
U32 matchIndex = hashTable[h];
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.cParams.chainLog - 1;
const U32 btLog = zc->appliedParams.cParams.chainLog - 1;
const U32 btMask= (1U << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const dictBase = zc->dictBase;
@ -267,7 +282,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit);
} else {
match = dictBase + matchIndex3;
if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
}
@ -410,10 +425,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
const BYTE* const base = ctx->base;
const BYTE* const prefixStart = base + ctx->dictLimit;
const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
const U32 sufficient_len = ctx->params.cParams.targetLength;
const U32 mls = ctx->params.cParams.searchLength;
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
const U32 mls = ctx->appliedParams.cParams.searchLength;
const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
ZSTD_match_t* matches = seqStorePtr->matchTable;
@ -439,7 +454,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
for (i=(ip == anchor); i<last_i; i++) {
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
@ -524,7 +539,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */
const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
@ -663,10 +678,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
const U32 sufficient_len = ctx->params.cParams.targetLength;
const U32 mls = ctx->params.cParams.searchLength;
const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
const U32 mls = ctx->appliedParams.cParams.searchLength;
const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
ZSTD_match_t* matches = seqStorePtr->matchTable;
@ -698,7 +713,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repMatch = repBase + repIndex;
if ( (repCur > 0 && repCur <= (S32)current)
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
@ -794,7 +809,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repMatch = repBase + repIndex;
if ( (repCur > 0 && repCur <= (S32)(current+cur))
&& (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
&& (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;

399
lib/compress/zstdmt_compress.c
View File

@ -14,34 +14,31 @@
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
/* ====== Dependencies ====== */
#include <stdlib.h> /* malloc */
#include <string.h> /* memcpy */
#include "pool.h" /* threadpool */
#include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include <string.h> /* memcpy, memset */
#include "pool.h" /* threadpool */
#include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include "zstdmt_compress.h"
/* ====== Debug ====== */
#if 0
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h>
# include <unistd.h>
# include <sys/times.h>
static unsigned g_debugLevel = 5;
# define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); }
# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
# define DEBUGLOGRAW(l, ...) if (l<=ZSTD_DEBUG) { fprintf(stderr, __VA_ARGS__); }
# define DEBUG_PRINTHEX(l,p,n) { \
unsigned debug_u; \
for (debug_u=0; debug_u<(n); debug_u++) \
# define DEBUG_PRINTHEX(l,p,n) { \
unsigned debug_u; \
for (debug_u=0; debug_u<(n); debug_u++) \
DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \
DEBUGLOGRAW(l, " \n"); \
DEBUGLOGRAW(l, " \n"); \
}
static unsigned long long GetCurrentClockTimeMicroseconds(void)
@ -54,21 +51,21 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
}
#define MUTEX_WAIT_TIME_DLEVEL 5
#define PTHREAD_MUTEX_LOCK(mutex) \
if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \
unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
pthread_mutex_lock(mutex); \
{ unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
unsigned long long const elapsedTime = (afterTime-beforeTime); \
if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \
DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
elapsedTime, #mutex); \
} } \
} else pthread_mutex_lock(mutex);
#define PTHREAD_MUTEX_LOCK(mutex) { \
if (ZSTD_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) { \
unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
pthread_mutex_lock(mutex); \
{ unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
unsigned long long const elapsedTime = (afterTime-beforeTime); \
if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \
DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
elapsedTime, #mutex); \
} } \
} else pthread_mutex_lock(mutex); \
}
#else
# define DEBUGLOG(l, ...) {} /* disabled */
# define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
# define DEBUG_PRINTHEX(l,p,n) {}
@ -87,16 +84,19 @@ static const buffer_t g_nullBuffer = { NULL, 0 };
typedef struct ZSTDMT_bufferPool_s {
unsigned totalBuffers;
unsigned nbBuffers;
ZSTD_customMem cMem;
buffer_t bTable[1]; /* variable size */
} ZSTDMT_bufferPool;
static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads)
static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_customMem cMem)
{
unsigned const maxNbBuffers = 2*nbThreads + 2;
ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t));
ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
if (bufPool==NULL) return NULL;
bufPool->totalBuffers = maxNbBuffers;
bufPool->nbBuffers = 0;
bufPool->cMem = cMem;
return bufPool;
}
@ -105,8 +105,21 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
unsigned u;
if (!bufPool) return; /* compatibility with free on NULL */
for (u=0; u<bufPool->totalBuffers; u++)
free(bufPool->bTable[u].start);
free(bufPool);
ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
ZSTD_free(bufPool, bufPool->cMem);
}
/* only works at initialization, not during compression */
static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
{
size_t const poolSize = sizeof(*bufPool)
+ (bufPool->totalBuffers - 1) * sizeof(buffer_t);
unsigned u;
size_t totalBufferSize = 0;
for (u=0; u<bufPool->totalBuffers; u++)
totalBufferSize += bufPool->bTable[u].size;
return poolSize + totalBufferSize;
}
/* assumption : invocation from main thread only ! */
@ -115,13 +128,15 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
if (pool->nbBuffers) { /* try to use an existing buffer */
buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
size_t const availBufferSize = buf.size;
if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) /* large enough, but not too much */
if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize))
/* large enough, but not too much */
return buf;
free(buf.start); /* size conditions not respected : scratch this buffer and create a new one */
/* size conditions not respected : scratch this buffer, create new one */
ZSTD_free(buf.start, pool->cMem);
}
/* create new buffer */
{ buffer_t buffer;
void* const start = malloc(bSize);
void* const start = ZSTD_malloc(bSize, pool->cMem);
if (start==NULL) bSize = 0;
buffer.start = start; /* note : start can be NULL if malloc fails ! */
buffer.size = bSize;
@ -138,7 +153,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
return;
}
/* Reached bufferPool capacity (should not happen) */
free(buf.start);
ZSTD_free(buf.start, pool->cMem);
}
@ -147,6 +162,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
typedef struct {
unsigned totalCCtx;
unsigned availCCtx;
ZSTD_customMem cMem;
ZSTD_CCtx* cctx[1]; /* variable size */
} ZSTDMT_CCtxPool;
@ -158,23 +174,40 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
unsigned u;
for (u=0; u<pool->totalCCtx; u++)
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
free(pool);
ZSTD_free(pool, pool->cMem);
}
/* ZSTDMT_createCCtxPool() :
* implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads)
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
ZSTD_customMem cMem)
{
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*));
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
if (!cctxPool) return NULL;
cctxPool->cMem = cMem;
cctxPool->totalCCtx = nbThreads;
cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
cctxPool->cctx[0] = ZSTD_createCCtx();
cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
DEBUGLOG(4, "cctxPool created, with %u threads", nbThreads);
return cctxPool;
}
/* only works during initialization phase, not during compression */
static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
{
unsigned const nbThreads = cctxPool->totalCCtx;
size_t const poolSize = sizeof(*cctxPool)
+ (nbThreads-1)*sizeof(ZSTD_CCtx*);
unsigned u;
size_t totalCCtxSize = 0;
for (u=0; u<nbThreads; u++)
totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
return poolSize + totalCCtxSize;
}
static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
{
if (pool->availCCtx) {
@ -218,7 +251,7 @@ typedef struct {
pthread_mutex_t* jobCompleted_mutex;
pthread_cond_t* jobCompleted_cond;
ZSTD_parameters params;
ZSTD_CDict* cdict;
const ZSTD_CDict* cdict;
unsigned long long fullFrameSize;
} ZSTDMT_jobDescription;
@ -228,11 +261,11 @@ void ZSTDMT_compressChunk(void* jobDescription)
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t const dstBuff = job->dstBuff;
DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
DEBUGLOG(4, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
if (job->cdict) { /* should only happen for first segment */
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
if (job->cdict) DEBUGLOG(3, "using CDict ");
DEBUGLOG(5, "using CDict");
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
@ -247,12 +280,12 @@ void ZSTDMT_compressChunk(void* jobDescription)
ZSTD_invalidateRepCodes(job->cctx);
}
DEBUGLOG(4, "Compressing : ");
DEBUGLOG(5, "Compressing : ");
DEBUG_PRINTHEX(4, job->srcStart, 12);
job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
DEBUGLOG(4, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
(unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
@ -271,6 +304,7 @@ _endJob:
struct ZSTDMT_CCtx_s {
POOL_ctx* factory;
ZSTDMT_jobDescription* jobs;
ZSTDMT_bufferPool* buffPool;
ZSTDMT_CCtxPool* cctxPool;
pthread_mutex_t jobCompleted_mutex;
@ -292,43 +326,51 @@ struct ZSTDMT_CCtx_s {
unsigned overlapRLog;
unsigned long long frameContentSize;
size_t sectionSize;
ZSTD_CDict* cdict;
ZSTD_CStream* cstream;
ZSTDMT_jobDescription jobs[1]; /* variable size (must lies at the end) */
ZSTD_customMem cMem;
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
};
ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
{
ZSTDMT_CCtx* cctx;
ZSTDMT_CCtx* mtctx;
U32 const minNbJobs = nbThreads + 2;
U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
U32 const nbJobs = 1 << nbJobsLog2;
DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n",
nbThreads, minNbJobs, nbJobsLog2, nbJobs);
DEBUGLOG(5, "nbThreads: %u ; minNbJobs: %u ; nbJobsLog2: %u ; nbJobs: %u",
nbThreads, minNbJobs, nbJobsLog2, nbJobs);
if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
if (!cctx) return NULL;
cctx->nbThreads = nbThreads;
cctx->jobIDMask = nbJobs - 1;
cctx->allJobsCompleted = 1;
cctx->sectionSize = 0;
cctx->overlapRLog = 3;
cctx->factory = POOL_create(nbThreads, 1);
cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) { /* one object was not created */
ZSTDMT_freeCCtx(cctx);
if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL))
/* invalid custom allocator */
return NULL;
mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem);
if (!mtctx) return NULL;
mtctx->cMem = cMem;
mtctx->nbThreads = nbThreads;
mtctx->jobIDMask = nbJobs - 1;
mtctx->allJobsCompleted = 1;
mtctx->sectionSize = 0;
mtctx->overlapRLog = 3;
mtctx->factory = POOL_create(nbThreads, 1);
mtctx->jobs = (ZSTDMT_jobDescription*)ZSTD_calloc(
nbJobs * sizeof(*mtctx->jobs), cMem);
mtctx->buffPool = ZSTDMT_createBufferPool(nbThreads, cMem);
mtctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads, cMem);
if (!mtctx->factory | !mtctx->jobs | !mtctx->buffPool | !mtctx->cctxPool) {
ZSTDMT_freeCCtx(mtctx);
return NULL;
}
if (nbThreads==1) {
cctx->cstream = ZSTD_createCStream();
if (!cctx->cstream) {
ZSTDMT_freeCCtx(cctx); return NULL;
} }
pthread_mutex_init(&cctx->jobCompleted_mutex, NULL); /* Todo : check init function return */
pthread_cond_init(&cctx->jobCompleted_cond, NULL);
DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
return cctx;
pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL); /* Todo : check init function return */
pthread_cond_init(&mtctx->jobCompleted_cond, NULL);
DEBUGLOG(4, "mt_cctx created, for %u threads", nbThreads);
return mtctx;
}
ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads)
{
return ZSTDMT_createCCtx_advanced(nbThreads, ZSTD_defaultCMem);
}
/* ZSTDMT_releaseAllJobResources() :
@ -356,15 +398,26 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
POOL_free(mtctx->factory);
if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
ZSTDMT_freeBufferPool(mtctx->buffPool); /* release job resources into pools first */
ZSTD_free(mtctx->jobs, mtctx->cMem);
ZSTDMT_freeCCtxPool(mtctx->cctxPool);
ZSTD_freeCDict(mtctx->cdict);
ZSTD_freeCStream(mtctx->cstream);
ZSTD_freeCDict(mtctx->cdictLocal);
pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
pthread_cond_destroy(&mtctx->jobCompleted_cond);
free(mtctx);
ZSTD_free(mtctx, mtctx->cMem);
return 0;
}
size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
{
if (mtctx == NULL) return 0; /* supports sizeof NULL */
return sizeof(*mtctx)
+ POOL_sizeof(mtctx->factory)
+ ZSTDMT_sizeof_bufferPool(mtctx->buffPool)
+ (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
+ ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
+ ZSTD_sizeof_CDict(mtctx->cdictLocal);
}
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
{
switch(parameter)
@ -373,7 +426,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
mtctx->sectionSize = value;
return 0;
case ZSTDMT_p_overlapSectionLog :
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
DEBUGLOG(5, "ZSTDMT_p_overlapSectionLog : %u", value);
mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
return 0;
default :
@ -404,8 +457,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */
size_t frameStartPos = 0, dstBufferPos = 0;
DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize);
DEBUGLOG(2, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
DEBUGLOG(4, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize);
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
params.fParams.contentSizeFlag = 1;
if (nbChunks==1) { /* fallback to single-thread mode */
@ -442,8 +495,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
DEBUGLOG(3, "posting job %u (%u bytes)", u, (U32)chunkSize);
DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
DEBUGLOG(4, "posting job %u (%u bytes)", u, (U32)chunkSize);
DEBUG_PRINTHEX(5, mtctx->jobs[u].srcStart, 12);
POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
frameStartPos += chunkSize;
@ -455,14 +508,14 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
{ unsigned chunkID;
size_t error = 0, dstPos = 0;
for (chunkID=0; chunkID<nbChunks; chunkID++) {
DEBUGLOG(3, "waiting for chunk %u ", chunkID);
DEBUGLOG(5, "waiting for chunk %u ", chunkID);
PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
while (mtctx->jobs[chunkID].jobCompleted==0) {
DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID);
pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
}
pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
DEBUGLOG(3, "ready to write chunk %u ", chunkID);
DEBUGLOG(5, "ready to write chunk %u ", chunkID);
ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
mtctx->jobs[chunkID].cctx = NULL;
@ -480,7 +533,7 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
dstPos += cSize ;
}
}
if (!error) DEBUGLOG(3, "compressed size : %u ", (U32)dstPos);
if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos);
return error ? error : dstPos;
}
@ -491,12 +544,13 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
/* ======= Streaming API ======= */
/* ====================================== */
static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
{
while (zcs->doneJobID < zcs->nextJobID) {
unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
while (zcs->jobs[jobID].jobCompleted==0) {
DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */
DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */
pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
}
pthread_mutex_unlock(&zcs->jobCompleted_mutex);
@ -505,33 +559,50 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
}
static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
const void* dict, size_t dictSize, unsigned updateDict,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
ZSTD_customMem const cmem = { NULL, NULL, NULL };
DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog);
if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize);
if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */
DEBUGLOG(5, "ZSTDMT_initCStream_internal");
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
if (zcs->nbThreads==1) {
DEBUGLOG(5, "single thread mode");
return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
dict, dictSize, cdict,
params, pledgedSrcSize);
}
if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(zcs);
ZSTDMT_releaseAllJobResources(zcs);
zcs->allJobsCompleted = 1;
}
zcs->params = params;
if (updateDict) {
ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
if (dict && dictSize) {
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
} }
zcs->frameContentSize = pledgedSrcSize;
if (dict) {
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byRef */, ZSTD_dm_auto,
params.cParams, zcs->cMem);
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = NULL;
zcs->cdict = cdict;
}
zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
DEBUGLOG(5, "overlapRLog : %u ", zcs->overlapRLog);
DEBUGLOG(5, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
DEBUGLOG(5, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
zcs->marginSize = zcs->targetSectionSize >> 2;
zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
@ -546,24 +617,39 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
return 0;
}
size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize);
DEBUGLOG(5, "ZSTDMT_initCStream_advanced");
return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, params, pledgedSrcSize);
}
size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams,
unsigned long long pledgedSrcSize)
{
ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */
params.fParams = fParams;
return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, cdict,
params, pledgedSrcSize);
}
/* ZSTDMT_resetCStream() :
* pledgedSrcSize is optional and can be zero == unknown */
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
{
if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize);
if (zcs->nbThreads==1)
return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize);
}
size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0);
return ZSTDMT_initCStream_internal(zcs, NULL, 0, NULL, params, 0);
}
@ -582,13 +668,16 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
return ERROR(memory_allocation);
}
DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ",
zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
zcs->jobs[jobID].src = zcs->inBuff.buffer;
zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
zcs->jobs[jobID].srcSize = srcSize;
zcs->jobs[jobID].dictSize = zcs->dictSize; /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */
zcs->jobs[jobID].dictSize = zcs->dictSize;
assert(zcs->inBuff.filled >= srcSize + zcs->dictSize);
zcs->jobs[jobID].params = zcs->params;
if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; /* do not calculate checksum within sections, just keep it in header for first section */
/* do not calculate checksum within sections, but write it in header for first section */
if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0;
zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
zcs->jobs[jobID].dstBuff = dstBuffer;
@ -603,6 +692,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
/* get a new buffer for next input */
if (!endFrame) {
size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
if (zcs->inBuff.buffer.start == NULL) { /* not enough memory to allocate next input buffer */
zcs->jobs[jobID].jobCompleted = 1;
@ -611,22 +701,33 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
ZSTDMT_releaseAllJobResources(zcs);
return ERROR(memory_allocation);
}
DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
DEBUGLOG(5, "inBuff currently filled to %u", (U32)zcs->inBuff.filled);
zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize));
memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled);
DEBUGLOG(5, "new job : inBuff filled to %u, with %u dict and %u src",
(U32)zcs->inBuff.filled, (U32)newDictSize,
(U32)(zcs->inBuff.filled - newDictSize));
memmove(zcs->inBuff.buffer.start,
(const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize,
zcs->inBuff.filled);
DEBUGLOG(5, "new inBuff pre-filled");
zcs->dictSize = newDictSize;
} else {
} else { /* if (endFrame==1) */
DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
zcs->inBuff.buffer = g_nullBuffer;
zcs->inBuff.filled = 0;
zcs->dictSize = 0;
zcs->frameEnded = 1;
if (zcs->nextJobID == 0)
zcs->params.fParams.checksumFlag = 0; /* single chunk : checksum is calculated directly within worker thread */
/* single chunk exception : checksum is calculated directly within worker thread */
zcs->params.fParams.checksumFlag = 0;
}
DEBUGLOG(3, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask);
DEBUGLOG(4, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)",
zcs->nextJobID,
(U32)zcs->jobs[jobID].srcSize,
zcs->jobs[jobID].lastChunk,
zcs->doneJobID,
zcs->doneJobID & zcs->jobIDMask);
POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]); /* this call is blocking when thread worker pool is exhausted */
zcs->nextJobID++;
return 0;
@ -664,7 +765,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */
U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
DEBUGLOG(4, "writing checksum : %08X \n", checksum);
DEBUGLOG(5, "writing checksum : %08X \n", checksum);
MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
job.cSize += 4;
zcs->jobs[wJobID].cSize += 4;
@ -675,7 +776,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
zcs->jobs[wJobID].jobScanned = 1;
}
{ size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
output->pos += toWrite;
job.dstFlushed += toWrite;
@ -699,23 +800,29 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
if (zcs->frameEnded) return ERROR(stage_wrong); /* current frame being ended. Only flush is allowed. Restart with init */
if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input);
if (zcs->frameEnded) {
/* current frame being ended. Only flush is allowed. Or start new job with init */
DEBUGLOG(5, "ZSTDMT_compressStream: zcs::frameEnded==1");
return ERROR(stage_wrong);
}
if (zcs->nbThreads==1) {
return ZSTD_compressStream(zcs->cctxPool->cctx[0], output, input);
}
/* fill input buffer */
{ size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad);
memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, (const char*)input->src + input->pos, toLoad);
input->pos += toLoad;
zcs->inBuff.filled += toLoad;
}
if ( (zcs->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */
&& (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { /* avoid overwriting job round buffer */
CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) );
CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0 /* endFrame */) );
}
/* check for data to flush */
CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */
CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize) /* blockToFlush */) ); /* block if it wasn't possible to create new job due to saturation */
/* recommended next input size : fill current input buffer */
return zcs->inBuffSize - zcs->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@ -726,26 +833,60 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
{
size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize);
if (srcSize)
DEBUGLOG(5, "ZSTDMT_flushStream_internal : %u bytes left to compress",
(U32)srcSize);
if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
&& (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
DEBUGLOG(5, "create new job with %u bytes to compress", (U32)srcSize);
DEBUGLOG(5, "end order : %u", endFrame);
CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
DEBUGLOG(5, "resulting zcs->frameEnded : %u", zcs->frameEnded);
}
/* check if there is any data available to flush */
DEBUGLOG(5, "zcs->doneJobID : %u ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID);
return ZSTDMT_flushNextJob(zcs, output, 1);
DEBUGLOG(5, "zcs->doneJobID : %u ; zcs->nextJobID : %u",
zcs->doneJobID, zcs->nextJobID);
return ZSTDMT_flushNextJob(zcs, output, 1 /*blockToFlush */);
}
size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
{
if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output);
return ZSTDMT_flushStream_internal(zcs, output, 0);
DEBUGLOG(5, "ZSTDMT_flushStream");
if (zcs->nbThreads==1)
return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */);
}
size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
{
if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output);
return ZSTDMT_flushStream_internal(zcs, output, 1);
DEBUGLOG(5, "ZSTDMT_endStream");
if (zcs->nbThreads==1)
return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */);
}
size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp)
{
DEBUGLOG(5, "in: pos:%u / size:%u ; endOp=%u",
(U32)input->pos, (U32)input->size, (U32)endOp);
if (input->pos < input->size) /* exclude final flushes */
CHECK_F(ZSTDMT_compressStream(mtctx, output, input));
if (input->pos < input->size) endOp = ZSTD_e_continue;
switch(endOp)
{
case ZSTD_e_flush:
return ZSTDMT_flushStream(mtctx, output);
case ZSTD_e_end:
DEBUGLOG(5, "endOp:%u; calling ZSTDMT_endStream", (U32)endOp);
return ZSTDMT_endStream(mtctx, output);
case ZSTD_e_continue:
return 1;
default:
return ERROR(GENERIC); /* invalid endDirective */
}
}

54
lib/compress/zstdmt_compress.h
View File

@ -15,25 +15,33 @@
#endif
/* Note : All prototypes defined in this file shall be considered experimental.
* There is no guarantee of API continuity (yet) on any of these prototypes */
/* Note : All prototypes defined in this file are labelled experimental.
* No guarantee of API continuity is provided on any of them.
* In fact, the expectation is that these prototypes will be replaced
* by ZSTD_compress_generic() API in the near future */
/* === Dependencies === */
#include <stddef.h> /* size_t */
#include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Simple one-pass functions === */
/* === Memory management === */
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
ZSTD_customMem cMem);
ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
/* === Simple buffer-to-butter one-pass function === */
ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
/* === Streaming functions === */
@ -53,8 +61,15 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
ZSTD_parameters params,
unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fparams,
unsigned long long pledgedSrcSize); /* note : zero means empty */
/* ZSDTMT_parameter :
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
@ -71,6 +86,19 @@ typedef enum {
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
/*! ZSTDMT_compressStream_generic() :
* Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
* depending on flush directive.
* @return : minimum amount of data still to be flushed
* 0 if fully flushed
* or an error code */
ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
#if defined (__cplusplus)
}
#endif

529
lib/decompress/zstd_decompress.c
View File

@ -53,8 +53,7 @@
# include "zstd_legacy.h"
#endif
#if defined(_MSC_VER)
#if defined(_MSC_VER) && !defined(_M_IA64) /* _mm_prefetch() is not defined for ia64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define ZSTD_PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
#elif defined(__GNUC__)
@ -63,8 +62,9 @@
# define ZSTD_PREFETCH(ptr) /* disabled */
#endif
/*-*************************************
* Macros
* Errors
***************************************/
#define ZSTD_isError ERR_isError /* for inlining */
#define FSE_isError ERR_isError
@ -85,6 +85,9 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef enum { zdss_init=0, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
typedef struct {
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
@ -117,11 +120,39 @@ struct ZSTD_DCtx_s
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
size_t staticSize;
/* streaming */
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict;
ZSTD_dStreamStage streamStage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
size_t lhSize;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
{
if (dctx==NULL) return 0; /* support sizeof NULL */
return sizeof(*dctx)
+ ZSTD_sizeof_DDict(dctx->ddictLocal)
+ dctx->inBuffSize + dctx->outBuffSize;
}
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
@ -145,36 +176,73 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
return 0;
}
static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
{
ZSTD_decompressBegin(dctx); /* cannot fail */
dctx->staticSize = 0;
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
dctx->ddict = NULL;
dctx->ddictLocal = NULL;
dctx->inBuff = NULL;
dctx->inBuffSize = 0;
dctx->outBuffSize= 0;
dctx->streamStage = zdss_init;
}
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx* dctx;
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
if (!dctx) return NULL;
dctx->customMem = customMem;
dctx->legacyContext = NULL;
ZSTD_initDCtx_internal(dctx);
return dctx;
}
}
dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx);
ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
{
ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace;
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
ZSTD_initDCtx_internal(dctx);
dctx->staticSize = workspaceSize;
dctx->inBuff = (char*)(dctx+1);
return dctx;
}
ZSTD_DCtx* ZSTD_createDCtx(void)
{
return ZSTD_createDCtx_advanced(defaultCustomMem);
return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
}
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
{
if (dctx==NULL) return 0; /* support free on NULL */
ZSTD_free(dctx, dctx->customMem);
return 0; /* reserved as a potential error code in the future */
if (dctx->staticSize) return ERROR(memory_allocation); /* not compatible with static DCtx */
{ ZSTD_customMem const cMem = dctx->customMem;
ZSTD_freeDDict(dctx->ddictLocal);
dctx->ddictLocal = NULL;
ZSTD_free(dctx->inBuff, cMem);
dctx->inBuff = NULL;
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (dctx->legacyContext)
ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
#endif
ZSTD_free(dctx, cMem);
return 0;
}
}
/* no longer useful */
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
}
@ -230,7 +298,8 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
/* skippable frame */
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
if (srcSize < ZSTD_skippableHeaderSize)
return ZSTD_skippableHeaderSize; /* magic number + frame length */
memset(zfhPtr, 0, sizeof(*zfhPtr));
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
zfhPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
@ -253,11 +322,13 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
U32 windowSize = 0;
U32 dictID = 0;
U64 frameContentSize = 0;
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
if ((fhdByte & 0x08) != 0)
return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
if (windowLog > ZSTD_WINDOWLOG_MAX)
return ERROR(frameParameter_windowTooLarge);
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@ -321,51 +392,48 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
* @return : decompressed size of the frames contained */
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{
{
unsigned long long totalDstSize = 0;
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
const U32 magicNumber = MEM_readLE32(src);
unsigned long long totalDstSize = 0;
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong);
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
const U32 magicNumber = MEM_readLE32(src);
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
continue;
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong);
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
{
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
/* check for overflow */
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += ret;
}
{
size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
if (ZSTD_isError(frameSrcSize)) {
return ZSTD_CONTENTSIZE_ERROR;
}
src = (const BYTE *)src + frameSrcSize;
srcSize -= frameSrcSize;
}
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
continue;
}
if (srcSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
{ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
return totalDstSize;
/* check for overflow */
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += ret;
}
{ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
if (ZSTD_isError(frameSrcSize)) {
return ZSTD_CONTENTSIZE_ERROR;
}
src = (const BYTE *)src + frameSrcSize;
srcSize -= frameSrcSize;
}
}
if (srcSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
return totalDstSize;
}
/** ZSTD_getDecompressedSize() :
@ -483,7 +551,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litCSize = (lhc >> 22) + (istart[4] << 10);
break;
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ?
@ -555,7 +623,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break;
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
@ -1288,7 +1356,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
@ -1503,6 +1571,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
size_t decodedSize;
size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
if (ZSTD_isError(frameSize)) return frameSize;
/* legacy support is incompatible with static dctx */
if (dctx->staticSize) return ERROR(memory_allocation);
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
@ -1578,7 +1648,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const
size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
size_t regenSize;
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
if (dctx==NULL) return ERROR(memory_allocation);
@ -1619,21 +1689,22 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
}
}
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
/** ZSTD_decompressContinue() :
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
* or an error code, which can be tested using ZSTD_isError() */
* srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
/* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
switch (dctx->stage)
{
case ZSTDds_getFrameHeaderSize :
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */
if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* unauthorized */
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
@ -1678,7 +1749,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->stage = ZSTDds_getFrameHeaderSize;
}
} else {
dctx->expected = 3; /* go directly to next header */
dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
dctx->stage = ZSTDds_decodeBlockHeader;
}
return 0;
@ -1904,33 +1975,39 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
}
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, unsigned byReference)
{
if ((byReference) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (!internalBuffer) return ERROR(memory_allocation);
memcpy(internalBuffer, dict, dictSize);
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_inDDict(ddict) );
return 0;
}
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (!ddict) return NULL;
ddict->cMem = customMem;
if ((byReference) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; }
memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, byReference) )) {
ZSTD_freeDDict(ddict);
return NULL;
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
{ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
if (ZSTD_isError(errorCode)) {
ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict;
}
@ -1946,7 +2023,6 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, to start decompression without startup delay.
* Dictionary content is simply referenced, it will be accessed during decompression.
@ -1958,6 +2034,24 @@ ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize
}
ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference)
{
size_t const neededSpace = sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace;
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
if (workspaceSize < neededSpace) return NULL;
if (!byReference) {
memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1;
}
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, 1 /* byRef */) ))
return NULL;
return ddict;
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
@ -1970,10 +2064,10 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
size_t ZSTD_estimateDDictSize(size_t dictSize)
* Note : dictionary created "byReference" are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference)
{
return dictSize + sizeof(ZSTD_DDict);
return sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
@ -2044,88 +2138,30 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
* Streaming decompression
*====================================*/
typedef enum { zdss_init, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
/* *** Resource management *** */
struct ZSTD_DStream_s {
ZSTD_DCtx* dctx;
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict;
ZSTD_frameHeader fParams;
ZSTD_dStreamStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
size_t lhSize;
ZSTD_customMem customMem;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
ZSTD_DStream* ZSTD_createDStream(void)
{
return ZSTD_createDStream_advanced(defaultCustomMem);
return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
}
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
{
ZSTD_DStream* zds;
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->dctx = ZSTD_createDCtx_advanced(customMem);
if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds;
return ZSTD_createDCtx_advanced(customMem);
}
size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->dctx);
zds->dctx = NULL;
ZSTD_freeDDict(zds->ddictLocal);
zds->ddictLocal = NULL;
ZSTD_free(zds->inBuff, cMem);
zds->inBuff = NULL;
ZSTD_free(zds->outBuff, cMem);
zds->outBuff = NULL;
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (zds->legacyContext)
ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
#endif
ZSTD_free(zds, cMem);
return 0;
}
return ZSTD_freeDCtx(zds);
}
/* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
zds->stage = zdss_loadHeader;
zds->streamStage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
ZSTD_freeDDict(zds->ddictLocal);
if (dict && dictSize >= 8) {
@ -2154,7 +2190,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict)
size_t ZSTD_resetDStream(ZSTD_DStream* zds)
{
zds->stage = zdss_loadHeader;
zds->streamStage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0;
zds->hostageByte = 0;
@ -2175,17 +2211,13 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support sizeof NULL */
return sizeof(*zds)
+ ZSTD_sizeof_DCtx(zds->dctx)
+ ZSTD_sizeof_DDict(zds->ddictLocal)
+ zds->inBuffSize + zds->outBuffSize;
return ZSTD_sizeof_DCtx(zds);
}
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
{
size_t const windowSize = fHeader.windowSize;
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
size_t const inBuffSize = blockSize; /* no block can be larger */
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
@ -2212,13 +2244,18 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
char* op = ostart;
U32 someMoreWork = 1;
DEBUGLOG(5, "ZSTD_decompressStream");
DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
if (zds->legacyVersion)
if (zds->legacyVersion) {
/* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation);
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
}
#endif
while (someMoreWork) {
switch(zds->stage)
switch(zds->streamStage)
{
case zdss_init :
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
@ -2226,22 +2263,25 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
case zdss_loadHeader :
{ size_t const hSize = ZSTD_getFrameHeader(&zds->fParams, zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize))
if (ZSTD_isError(hSize)) {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
/* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation);
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else {
return hSize; /* error */
} }
}
#else
return hSize;
#endif
}
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
@ -2252,82 +2292,97 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
break;
}
} }
/* check for single-pass mode opportunity */
if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
if (cSize <= (size_t)(iend-istart)) {
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize;
ip = istart + cSize;
op += decompressedSize;
zds->dctx->expected = 0;
zds->stage = zdss_init;
someMoreWork = 0;
break;
} }
/* check for single-pass mode opportunity */
if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
if (cSize <= (size_t)(iend-istart)) {
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
if (ZSTD_isError(decompressedSize)) return decompressedSize;
ip = istart + cSize;
op += decompressedSize;
zds->expected = 0;
zds->streamStage = zdss_init;
someMoreWork = 0;
break;
} }
/* Consume header */
CHECK_F(ZSTD_decompressBegin_usingDDict(zds->dctx, zds->ddict));
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
} }
/* Consume header (see ZSTDds_decodeFrameHeader) */
DEBUGLOG(5, "Consume header");
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
zds->expected = MEM_readLE32(zds->headerBuffer + 4);
zds->stage = ZSTDds_skipFrame;
} else {
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
zds->expected = ZSTD_blockHeaderSize;
zds->stage = ZSTDds_decodeBlockHeader;
}
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) {
/* control buffer memory usage */
DEBUGLOG(5, "Control max buffer memory usage");
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
zds->blockSize = blockSize;
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
size_t const bufferSize = blockSize + neededOutSize;
DEBUGLOG(5, "inBuff : from %u to %u",
(U32)zds->inBuffSize, (U32)blockSize);
DEBUGLOG(5, "outBuff : from %u to %u",
(U32)zds->outBuffSize, (U32)neededOutSize);
if (zds->staticSize) { /* static DCtx */
DEBUGLOG(5, "staticSize : %u", (U32)zds->staticSize);
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
return ERROR(memory_allocation);
} else {
ZSTD_free(zds->inBuff, zds->customMem);
zds->inBuffSize = 0;
zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
zds->inBuffSize = blockSize;
}
if (zds->outBuffSize < neededOutSize) {
ZSTD_free(zds->outBuff, zds->customMem);
zds->outBuffSize = 0;
zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem);
if (zds->outBuff == NULL) return ERROR(memory_allocation);
zds->outBuffSize = neededOutSize;
} }
zds->stage = zdss_read;
}
zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
}
zds->inBuffSize = blockSize;
zds->outBuff = zds->inBuff + zds->inBuffSize;
zds->outBuffSize = neededOutSize;
} }
zds->streamStage = zdss_read;
/* pass-through */
case zdss_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
DEBUGLOG(5, "stage zdss_read");
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init;
zds->streamStage = zdss_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
int const isSkipFrame = ZSTD_isSkipFrame(zds);
size_t const decodedSize = ZSTD_decompressContinue(zds,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
zds->streamStage = zdss_flush;
break;
}
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->stage = zdss_load;
/* pass-through */
}
} }
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->streamStage = zdss_load;
/* pass-through */
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
@ -2337,17 +2392,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
{ const int isSkipFrame = ZSTD_isSkipFrame(zds);
size_t const decodedSize = ZSTD_decompressContinue(zds,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */
if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
/* pass-through */
} }
zds->streamStage = zdss_flush;
/* pass-through */
case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
@ -2355,37 +2410,41 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
op += flushedSize;
zds->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */
zds->stage = zdss_read;
zds->streamStage = zdss_read;
if (zds->outStart + zds->blockSize > zds->outBuffSize)
zds->outStart = zds->outEnd = 0;
break;
}
/* cannot complete flush */
someMoreWork = 0;
break;
}
} }
/* cannot complete flush */
someMoreWork = 0;
break;
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
if (!nextSrcSizeHint) { /* frame fully decoded */
if (zds->outEnd == zds->outStart) { /* output fully flushed */
if (zds->hostageByte) {
if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */
if (input->pos >= input->size) {
/* can't release hostage (not present) */
zds->streamStage = zdss_read;
return 1;
}
input->pos++; /* release hostage */
}
} /* zds->hostageByte */
return 0;
}
} /* zds->outEnd == zds->outStart */
if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
zds->hostageByte=1;
}
return 1;
}
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
} /* nextSrcSizeHint==0 */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;

8
lib/dictBuilder/zdict.c
View File

@ -576,7 +576,7 @@ typedef struct
{
ZSTD_CCtx* ref;
ZSTD_CCtx* zc;
void* workPlace; /* must be ZSTD_BLOCKSIZE_ABSOLUTEMAX allocated */
void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
} EStats_ress_t;
#define MAXREPOFFSET 1024
@ -585,14 +585,14 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize, U32 notificationLevel)
{
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
@ -700,7 +700,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
/* init */
esr.ref = ZSTD_createCCtx();
esr.zc = ZSTD_createCCtx();
esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
if (!esr.ref || !esr.zc || !esr.workPlace) {
eSize = ERROR(memory_allocation);
DISPLAYLEVEL(1, "Not enough memory \n");

448
lib/zstd.h
View File

@ -38,20 +38,21 @@ extern "C" {
/*******************************************************************************************************
Introduction
zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
zstd, short for Zstandard, is a fast lossless compression algorithm,
targeting real-time compression scenarios at zlib-level and better compression ratios.
The zstd compression library provides in-memory compression and decompression functions.
The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
Compression can be done in:
- a single step (described as Simple API)
- a single step, reusing a context (described as Explicit memory management)
- unbounded multiple steps (described as Streaming compression)
The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
The compression ratio achievable on small data can be highly improved using a dictionary in:
- a single step (described as Simple dictionary API)
- a single step, reusing a dictionary (described as Fast dictionary API)
Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
These APIs shall never be used with a dynamic library.
Advanced experimental APIs shall never be used with a dynamic library.
They are not "stable", their definition may change in the future. Only static linking is allowed.
*********************************************************************************************************/
@ -61,13 +62,13 @@ extern "C" {
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to be used when checking dll version */
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
ZSTDLIB_API const char* ZSTD_versionString(void);
ZSTDLIB_API const char* ZSTD_versionString(void); /* >= v1.3.0 */
/***************************************
@ -84,7 +85,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
/*! ZSTD_decompress() :
* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
* `dstCapacity` is an upper bound of originalSize.
* `dstCapacity` is an upper bound of originalSize to regenerate.
* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
* or an errorCode if it fails (which can be tested using ZSTD_isError()). */
@ -92,21 +93,17 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*! ZSTD_getDecompressedSize() :
* NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
* ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
* frame, but distinguishes empty frames from frames with an unknown size, or errors.
*
* Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
* concatenated frames in one buffer, and so is more general.
* As a result however, it requires more computation and entire frames to be passed to it,
* as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
* NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize().
* ZSTD_getFrameContentSize() works the same way,
* returning the decompressed size of a single frame,
* but distinguishes empty frames from frames with an unknown size, or errors.
*
* 'src' is the start of a zstd compressed frame.
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
* note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode.
* When `return==0`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
* Optionally, application can use ZSTD_decompress() while relying on implied limits.
* (For example, data may be necessarily cut into blocks <= 16 KB).
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
* note 3 : decompressed size can be very large (64-bits value),
@ -115,7 +112,7 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameHeader() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
@ -140,29 +137,35 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/*! ZSTD_compressCCtx() :
* Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
/*= Decompression context
* When decompressing many times,
* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
* it is recommended to allocate a context only once,
* and re-use it for each successive compression operation.
* This will make workload friendlier for system's memory.
* Use one context per thread for parallel execution in multi-threaded environments. */
* Use one context per thread for parallel execution. */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
/*! ZSTD_decompressDCtx() :
* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/**************************
* Simple dictionary API
***************************/
/*! ZSTD_compress_usingDict() :
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
* Note : This function loads the dictionary, resulting in significant startup delay.
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
* Note : This function loads the dictionary, resulting in significant startup delay.
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -170,30 +173,31 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
int compressionLevel);
/*! ZSTD_decompress_usingDict() :
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
* Note : This function loads the dictionary, resulting in significant startup delay.
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
* Note : This function loads the dictionary, resulting in significant startup delay.
* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
/****************************
* Fast dictionary API
****************************/
/**********************************
* Bulk processing dictionary API
*********************************/
typedef struct ZSTD_CDict_s ZSTD_CDict;
/*! ZSTD_createCDict() :
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
* `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
/*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict(). */
* Function frees memory allocated by ZSTD_createCDict(). */
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() :
@ -210,17 +214,17 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* dictBuffer can be released after DDict creation, as its content is copied inside DDict */
* Create a digested dictionary, ready to start decompression operation without startup delay.
* dictBuffer can be released after DDict creation, as its content is copied inside DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */
* Function frees memory allocated with ZSTD_createDDict() */
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() :
* Decompression using a digested Dictionary.
* Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
* Decompression using a digested Dictionary.
* Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -277,15 +281,17 @@ typedef struct ZSTD_outBuffer_s {
* ZSTD_endStream() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
* ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush.
* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
* @return : 0 if frame fully completed and fully flushed,
or >0 if some data is still present within internal buffer
(value is minimum size estimation for remaining data to flush, but it could be more)
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************/
typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are effectively same object */
/* Continue due distinghish them for compatibility with versions <= v1.2.0 */
typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
/* But continue to distinguish them for compatibility with versions <= v1.2.0 */
/*===== ZSTD_CStream management functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@ -323,7 +329,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
/* But continue to distinguish them for compatibility with versions <= v1.2.0 */
/*===== ZSTD_DStream management functions =====*/
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@ -338,8 +345,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#endif /* ZSTD_H_235446 */
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/****************************************************************************************
* START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
@ -349,6 +354,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
* Use them only in association with static linking.
* ***************************************************************************************/
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
@ -378,7 +386,8 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f
/*--- Advanced types ---*/
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */
typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@ -412,6 +421,9 @@ typedef struct {
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/* use this constant to defer to stdlib's functions */
static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
/***************************************
* Frame size functions
@ -485,21 +497,22 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* These functions make it possible to estimate memory usage
* of a future target object, before its allocation,
* given a set of parameters, which vary depending on target object.
* The objective is to guide decision before allocation. */
* The objective is to guide decision before allocation.
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
/*! ZSTD_estimate?StreamSize() :
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
* an internal ?Dict will be created, which size is not estimated.
* In this case, get additional size by using ZSTD_estimate?DictSize */
* an internal ?Dict will be created, which size is not estimated here.
* In this case, get total size by adding ZSTD_estimate?DictSize */
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
/*! ZSTD_estimate?DictSize() :
* Note : if dictionary is created "byReference", reduce estimation by dictSize */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
* Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
/***************************************
@ -509,6 +522,25 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
* workspace: The memory area to emplace the context into.
* Provided pointer must 8-bytes aligned.
* It must outlive context usage.
* workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
* to determine how large workspace must be to support scenario.
* @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
* Note : zstd will never resize nor malloc() when using a static cctx.
* If it needs more memory than available, it will simply error out.
* Note 2 : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally too.
* Limitation 1 : currently not compatible with internal CDict creation, such as
* ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
* Limitation 2 : currently not compatible with multi-threading
*/
ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
/* !!! To be deprecated !!! */
typedef enum {
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@ -518,17 +550,40 @@ typedef enum {
* @result : 0, or an error code (which can be tested with ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
/*! ZSTD_initStaticCDict_advanced() :
* Generate a digested dictionary in provided memory area.
* workspace: The memory area to emplace the dictionary into.
* Provided pointer must 8-bytes aligned.
* It must outlive dictionary usage.
* workspaceSize: Use ZSTD_estimateCDictSize()
* to determine how large workspace must be.
* cParams : use ZSTD_getCParams() to transform a compression level
* into its relevants cParams.
* @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
* Note : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
*/
ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams);
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
* `estimatedSrcSize` value is optional, select 0 if not known */
@ -544,8 +599,8 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long l
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
/*! ZSTD_adjustCParams() :
* optimize params for a given `srcSize` and `dictSize`.
* both values are optional, select `0` if unknown. */
* optimize params for a given `srcSize` and `dictSize`.
* both values are optional, select `0` if unknown. */
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() :
@ -577,10 +632,28 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
* workspace: The memory area to emplace the context into.
* Provided pointer must 8-bytes aligned.
* It must outlive context usage.
* workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
* to determine how large workspace must be to support scenario.
* @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
* Note : zstd will never resize nor malloc() when using a static dctx.
* If it needs more memory than available, it will simply error out.
* Note 2 : static dctx is incompatible with legacy support
* Note 3 : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
* Limitation : currently not compatible with internal DDict creation,
* such as ZSTD_initDStream_usingDict().
*/
ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
* Dictionary content is referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict,
* it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
/*! ZSTD_createDDict_advanced() :
@ -588,6 +661,21 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_initStaticDDict() :
* Generate a digested dictionary in provided memory area.
* workspace: The memory area to emplace the dictionary into.
* Provided pointer must 8-bytes aligned.
* It must outlive dictionary usage.
* workspaceSize: Use ZSTD_estimateDDictSize()
* to determine how large workspace must be.
* @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
* Note : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
*/
ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize,
unsigned byReference);
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
@ -609,7 +697,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
* Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */
* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
@ -620,11 +708,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
/*! ZSTD_resetCStream() :
* start a new compression job, using same parameters from previous job.
@ -704,7 +792,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
and the dictionary ID used.
@ -751,7 +839,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_decompressContinue() always returns 0.
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
For purposes of decompression, it is valid in both cases to skip the frame using
ZSTD_findFrameCompressedSize to find its size in bytes.
@ -770,6 +858,221 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/*=== New advanced API (experimental, and compression only) ===*/
/* notes on API design :
* In this proposal, parameters are pushed one by one into an existing CCtx,
* and then applied on all subsequent compression jobs.
* When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
*
* This API is intended to replace all others experimental API.
* It can basically do all other use cases, and even new ones.
* It stands a good chance to become "stable",
* after a reasonable testing period.
*/
/* note on naming convention :
* Initially, the API favored names like ZSTD_setCCtxParameter() .
* In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
* The main driver is that it identifies more clearly the target object type.
* It feels clearer in light of potential variants :
* ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
* ZSTD_DCtx_setParameter() (rather than ZSTD_setDCtxParameter() )
* Left variant feels easier to distinguish.
*/
/* note on enum design :
* All enum will be manually set to explicit values before reaching "stable API" status */
typedef enum {
/* compression parameters */
ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means "do not change cLevel". */
ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "do not change windowLog". */
ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
* Resulting table size is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
* Larger tables improve compression ratio of strategies <= dFast,
* and improve speed of strategies > dFast.
* Special: value 0 means "do not change hashLog". */
ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2.
* Resulting table size is (1 << (chainLog+2)).
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* Special: value 0 means "do not change chainLog". */
ZSTD_p_searchLog, /* Number of search attempts, as a power of 2.
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* Special: value 0 means "do not change searchLog". */
ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller).
* Larger values make faster compression and decompression, but decrease ratio.
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
* Note that currently, for all strategies < btopt, effective minimum is 4.
* Note that currently, for all strategies > fast, effective maximum is 6.
* Special: value 0 means "do not change minMatchLength". */
ZSTD_p_targetLength, /* Only useful for strategies >= btopt.
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger and slower.
* Special: value 0 means "do not change targetLength". */
ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
#if 0
ZSTD_p_windowSize, /* Maximum allowed back-reference distance.
* Can be set to a more precise value than windowLog.
* Will be transparently reduced to closest possible inferior value
* (see Zstandard compression format) */
/* Not ready yet ! */
#endif
/* frame parameters */
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
/* dictionary parameters */
ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
* This avoids duplicating dictionary content.
* But it also requires that dictionary buffer outlives its users */
/* Not ready yet ! <=================================== */
ZSTD_p_dictMode, /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
* More threads improve speed, but also increase memory usage.
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
* Special: value 0 means "do not change nbThreads" */
ZSTD_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */
} ZSTD_cParameter;
/*! ZSTD_CCtx_setParameter() :
* Set one compression parameter, selected by enum ZSTD_cParameter.
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
* @result : 0, or an error code (which can be tested with ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
/*! ZSTD_CCtx_setPledgedSrcSize() :
* Total input data size to be compressed as a single frame.
* This value will be controlled at the end, and result in error if not respected.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Note 1 : 0 means zero, empty.
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
* Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
* Note 2 : If all data is provided and consumed in a single round,
* this value is overriden by srcSize instead. */
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
/*! ZSTD_CCtx_loadDictionary() :
* Create an internal CDict from dict buffer.
* Decompression will have to use same buffer.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
* meaning "return to no-dictionary mode".
* Note 1 : Dictionary content will be copied internally,
* except if ZSTD_p_refDictContent is set.
* Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
* For this reason, compression parameters cannot be changed anymore after loading a dictionary.
* It's also a CPU-heavy operation, with non-negligible impact on latency.
* Note 3 : Dictionary will be used for all future compression jobs.
* To return to "no-dictionary" situation, load a NULL dictionary */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() :
* Ref a prepared dictionary, to be used for all next compression jobs.
* Note that compression parameters are enforced from within CDict,
* and supercede any compression parameter previously set within CCtx.
* The dictionary will remain valid for future compression jobs using same CCtx.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : adding a NULL CDict means "return to no-dictionary mode".
* Note 1 : Currently, only one dictionary can be managed.
* Adding a new dictionary effectively "discards" any previous one.
* Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
*/
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
/*! ZSTD_CCtx_refPrefix() :
* Reference a prefix (content-only dictionary) to bootstrap next compression job.
* Decompression will have to use same prefix.
* Prefix is only used once. Tables are discarded at end of compression job.
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
* Note 1 : Prefix buffer is referenced. It must outlive compression job.
* Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU-heavy operation, with non-negligible impact on latency. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); /* Not ready yet ! <===================================== */
typedef enum {
ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
ZSTD_e_end /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
} ZSTD_EndDirective;
/*! ZSTD_compress_generic() :
* Behave about the same as ZSTD_compressStream. To note :
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
* - Compression parameters cannot be changed once compression is started.
* - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
* - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
* - @return provides the minimum amount of data still to flush from internal buffers
* or an error code, which can be tested using ZSTD_isError().
* if @return != 0, flush is not fully completed, there is some data left within internal buffers.
* - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
* It is necessary to fully flush internal buffers
* before starting a new compression job, or changing compression parameters.
*/
ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
/*! ZSTD_CCtx_reset() :
* Return a CCtx to clean state.
* Useful after an error, or to interrupt an ongoing compression job and start a new one.
* Any internal data not yet flushed is cancelled.
* Dictionary (if any) is dropped.
* It's possible to modify compression parameters after a reset.
*/
ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); /* Not ready yet ! */
/*! ZSTD_compress_generic_simpleArgs() :
* Same as ZSTD_compress_generic(),
* but using only integral types as arguments.
* Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
* but can be helpful for binders from dynamic languages
* which have troubles handling structures containing memory pointers.
*/
size_t ZSTD_compress_generic_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos,
ZSTD_EndDirective endOp);
/**
Block functions
@ -784,7 +1087,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+ copyCCtx() and copyDCtx() can be used too
- Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@ -797,9 +1100,10 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
Use ZSTD_insertBlock() for such a case.
*/
#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
#define ZSTD_BLOCKSIZELOG_MAX 17
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
/*===== Raw zstd block functions =====*/
ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */

68
programs/bench.c
View File

@ -86,14 +86,13 @@ static clock_t g_time = 0;
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#define EXM_THROW(error, ...) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
}
@ -159,7 +158,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const ZSTD_compressionParameters* comprParams)
{
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
@ -262,42 +260,72 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart);
if (!cCompleted) { /* still some time to do compression tests */
ZSTD_customMem const cmem = { NULL, NULL, NULL };
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
ZSTD_CDict* cdict = NULL;
#ifdef ZSTD_NEWAPI
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
#else
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_CDict* cdict;
ZSTD_customMem const cmem = { NULL, NULL, NULL };
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1 /*byRef*/, ZSTD_dm_auto, zparams.cParams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
#endif
do {
U32 blockNb;
size_t rSize;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize;
#ifdef ZSTD_NEWAPI
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
size_t cError = 1;
while (cError) {
cError = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end);
if (ZSTD_isError(cError))
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
ZSTD_getErrorName(cError));
}
rSize = out.pos;
#else /* ! ZSTD_NEWAPI */
if (dictBufferSize) {
rSize = ZSTD_compress_usingCDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
} else {
#ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
rSize = ZSTDMT_compressCCtx(mtctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cLevel);
#else
# else
rSize = ZSTD_compress_advanced (ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
#endif
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
NULL, 0, zparams);
# endif
}
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
if (ZSTD_isError(rSize))
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
ZSTD_getErrorName(rSize));
#endif /* ZSTD_NEWAPI */
blockTable[blockNb].cSize = rSize;
}
nbLoops++;
@ -383,7 +411,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
if (u>5) {
int n;
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);

224
programs/fileio.c
View File

@ -77,10 +77,7 @@
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0
#define CACHELINE 64
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
@ -93,7 +90,7 @@
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
@ -103,10 +100,35 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
#undef MIN
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
/*-*************************************
* Errors
***************************************/
#ifndef ZSTD_DEBUG
# define ZSTD_DEBUG 0
#endif
#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DISPLAYLEVEL(1, "zstd: "); \
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
}
#define CHECK(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
DEBUGLOG(1, "%s \n", #f); \
EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
} }
/* ************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
***************************************************************/
@ -146,7 +168,7 @@ static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; }
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
@ -182,23 +204,6 @@ void FIO_setOverlapLog(unsigned overlapLog){
}
/*-*************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, " \n"); \
exit(error); \
}
/*-*************************************
* Functions
***************************************/
@ -226,12 +231,14 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
f = stdin;
SET_BINARY_MODE(stdin);
} else {
if (!UTIL_isRegFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName);
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
f = fopen(srcFileName, "rb");
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
if ( f==NULL )
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
}
return f;
@ -256,26 +263,28 @@ static FILE* FIO_openDstFile(const char* dstFileName)
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
if (strcmp (dstFileName, nulmark)) {
/* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
if (f != 0) { /* dst file exists, prompt for overwrite authorization */
fclose(f);
if (!g_overwrite) {
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
dstFileName);
{ int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return NULL;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
}
}
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} }
/* need to unlink */
FIO_remove(dstFileName);
} }
@ -303,11 +312,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
if (fileSize > DICTSIZE_MAX)
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
fclose(fileHandle);
@ -326,7 +337,7 @@ typedef struct {
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
#ifdef ZSTD_MULTITHREAD
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_CCtx* cctx;
#else
ZSTD_CStream* cctx;
@ -334,15 +345,19 @@ typedef struct {
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
U64 srcSize, int srcRegFile,
U64 srcSize, int srcIsRegularFile,
ZSTD_compressionParameters* comprParams) {
cRess_t ress;
memset(&ress, 0, sizeof(ress));
#ifdef ZSTD_MULTITHREAD
#ifdef ZSTD_NEWAPI
ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
#elif defined(ZSTD_MULTITHREAD)
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
if (ress.cctx == NULL)
EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
/* use complete window for overlap */
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
@ -351,22 +366,44 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
#else
ress.cctx = ZSTD_createCStream();
if (ress.cctx == NULL)
EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
#endif
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer)
EXM_THROW(31, "zstd: allocation error : not enough memory");
EXM_THROW(31, "allocation error : not enough memory");
/* dictionary */
{ void* dictBuffer;
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
EXM_THROW(32, "allocation error : can't create dictBuffer");
#ifdef ZSTD_NEWAPI
{ /* frame parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, srcIsRegularFile) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );
/* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
/* multi-threading */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.contentSizeFlag = srcRegFile;
params.fParams.contentSizeFlag = srcIsRegularFile;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
@ -376,15 +413,24 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
#ifdef ZSTD_MULTITHREAD
{ size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
}
#else
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.contentSizeFlag = srcIsRegularFile;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
}
#endif
} }
free(dictBuffer);
}
@ -395,7 +441,7 @@ static void FIO_freeCResources(cRess_t ress)
{
free(ress.srcBuffer);
free(ress.dstBuffer);
#ifdef ZSTD_MULTITHREAD
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_freeCCtx(ress.cctx);
#else
ZSTD_freeCStream(ress.cctx); /* never fails */
@ -700,41 +746,40 @@ static int FIO_compressFilename_internal(cRess_t ress,
}
/* init */
#ifdef ZSTD_MULTITHREAD
{ size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize);
#ifdef ZSTD_NEWAPI
/* nothing, reset is implied */
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );
#else
{ size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
CHECK( ZSTD_resetCStream(ress.cctx, fileSize) );
#endif
if (ZSTD_isError(resetError))
EXM_THROW(21, "Error initializing compression : %s",
ZSTD_getErrorName(resetError));
}
/* Main compression loop */
while (1) {
/* Fill input Buffer */
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
if (inSize==0) break;
readsize += inSize;
{ ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
while (inBuff.pos != inBuff.size) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_MULTITHREAD
size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
while (inBuff.pos != inBuff.size) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
CHECK( ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_continue) );
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
#else
size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
#endif
if (ZSTD_isError(result))
EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result));
/* Write compressed stream */
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos)
EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += outBuff.pos;
} } }
/* Write compressed stream */
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos)
EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += outBuff.pos;
} }
if (g_nbThreads > 1) {
if (!fileSize)
DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
@ -757,12 +802,18 @@ static int FIO_compressFilename_internal(cRess_t ress,
{ size_t result = 1;
while (result!=0) { /* note : is there any possibility of endless loop ? */
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_MULTITHREAD
#ifdef ZSTD_NEWAPI
ZSTD_inBuffer inBuff = { NULL, 0, 0};
result = ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_end);
#elif defined(ZSTD_MULTITHREAD)
result = ZSTDMT_endStream(ress.cctx, &outBuff);
#else
result = ZSTD_endStream(ress.cctx, &outBuff);
#endif
if (ZSTD_isError(result)) EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result));
if (ZSTD_isError(result))
EXM_THROW(26, "Compression error during frame end : %s",
ZSTD_getErrorName(result));
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += outBuff.pos;
@ -851,9 +902,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
{
clock_t const start = clock();
U64 const srcSize = UTIL_getFileSize(srcFileName);
int const regFile = UTIL_isRegFile(srcFileName);
int const isRegularFile = UTIL_isRegularFile(srcFileName);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@ -1098,8 +1149,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
/* init */
if (dstFileName==NULL)
@ -1177,9 +1228,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* dictionary */
{ void* dictBuffer;
size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
if (ZSTD_isError(initError))
EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
free(dictBuffer);
}
@ -1188,10 +1237,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
static void FIO_freeDResources(dRess_t ress)
{
size_t const errorCode = ZSTD_freeDStream(ress.dctx);
if (ZSTD_isError(errorCode))
EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s",
ZSTD_getErrorName(errorCode));
CHECK( ZSTD_freeDStream(ress.dctx) );
free(ress.srcBuffer);
free(ress.dstBuffer);
}

2
programs/util.h
View File

@ -208,7 +208,7 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
}
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */

BIN
programs/windres/zstd32.res
View File

Binary file not shown.

BIN
programs/windres/zstd64.res
View File

Binary file not shown.

6
programs/zstd.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "May 2017" "zstd 1.3.0" "User Commands"
.TH "ZSTD" "1" "June 2017" "zstd 1.3.0" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -235,8 +235,8 @@ benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\
minimum evaluation time, in seconds (default : 3s), benchmark mode only
.
.TP
\fB\-B#\fR
cut file into independent blocks of size # (default: no block)
\fB\-B#\fR, \fB\-\-block\-size=#\fR
cut file(s) into independent blocks of size # (default: no block)
.
.TP
\fB\-\-priority=rt\fR

8
programs/zstd.1.md
View File

@ -234,8 +234,8 @@ BENCHMARK
benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
* `-i#`:
minimum evaluation time, in seconds (default : 3s), benchmark mode only
* `-B#`:
cut file into independent blocks of size # (default: no block)
* `-B#`, `--block-size=#`:
cut file(s) into independent blocks of size # (default: no block)
* `--priority=rt`:
set process priority to real-time
@ -255,8 +255,8 @@ The list of available _options_:
Specify a strategy used by a match finder.
There are 8 strategies numbered from 0 to 7, from faster to stronger:
0=ZSTD\_fast, 1=ZSTD\_dfast, 2=ZSTD\_greedy, 3=ZSTD\_lazy,
4=ZSTD\_lazy2, 5=ZSTD\_btlazy2, 6=ZSTD\_btopt, 7=ZSTD\_btultra.
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.

2
programs/zstdcli.c
View File

@ -300,7 +300,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
return 0;
}

24
tests/Makefile
View File

@ -26,18 +26,18 @@ PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
DEBUGFLAGS=-g -DZSTD_DEBUG=1
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
$(DEBUGFLAG)
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
DEBUGFLAGS= -g -DZSTD_DEBUG=1
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wstrict-prototypes -Wundef -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
CFLAGS += $(DEBUGFLAGS)
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@ -159,7 +159,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAG =
paramgrill : DEBUGFLAGS =
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
@ -321,6 +321,8 @@ test-zbuff32: zbufftest32
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --mt $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)

2
tests/decodecorpus.c
View File

@ -73,8 +73,6 @@ static clock_t clockSpan(clock_t cStart)
/*-*******************************************************
* Random function
*********************************************************/
#define CLAMP(x, a, b) ((x) < (a) ? (a) : ((x) > (b) ? (b) : (x)))
static unsigned RAND(unsigned* src)
{
#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))

202
tests/fuzzer.c
View File

@ -12,9 +12,9 @@
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
@ -25,7 +25,7 @@
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zstdmt_compress.h"
@ -74,7 +74,6 @@ static clock_t FUZ_clockSpan(clock_t cStart)
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
}
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned FUZ_rand(unsigned* src)
{
@ -104,6 +103,7 @@ static unsigned FUZ_highbit32(U32 v32)
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
#define CHECK(fn) { CHECK_V(err, fn); }
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
static int basicUnitTests(U32 seed, double compressibility)
{
size_t const CNBuffSize = 5 MB;
@ -190,6 +190,90 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
void* const staticCCtxBuffer = malloc(staticCCtxSize);
size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
void* const staticDCtxBuffer = malloc(staticDCtxSize);
if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
free(staticCCtxBuffer);
free(staticDCtxBuffer);
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
{ ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
{ size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++);
CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n",
(U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++);
{ size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u])
goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
{ size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
{ size_t const r = ZSTD_compressBegin(staticCCtx, 1);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CStream for small level %u : ", testNb++, 1);
{ size_t const r = ZSTD_initCStream(staticCCtx, 1);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
{ size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init DStream (should fail) : ", testNb++);
{ size_t const r = ZSTD_initDStream(staticDCtx);
if (ZSTD_isError(r)) goto _output_error; }
{ ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
}
/* ZSTDMT simple MT compression test */
DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
{ ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
@ -321,13 +405,25 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
{ ZSTD_DDict* const ddict = ZSTD_createDDict_byReference(CNBuffer, dictSize);
{ ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
if (r != CNBuffSize - dictSize) goto _output_error;
DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
ZSTD_freeDDict(ddict);
}
DISPLAYLEVEL(4, "test%3i : decompress with static DDict : ", testNb++);
{ size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, 0);
void* ddictBuffer = malloc(ddictBufferSize);
if (ddictBuffer == NULL) goto _output_error;
{ ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, 0);
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
if (r != CNBuffSize - dictSize) goto _output_error;
}
free(ddictBuffer);
DISPLAYLEVEL(4, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize);
}
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3;
{ ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
@ -406,14 +502,16 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1 /*byReference*/);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
}
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
DISPLAYLEVEL(4, "test%3i : compress with CDict ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem customMem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
1 /* byReference */, ZSTD_dm_auto,
cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict);
ZSTD_freeCDict(cdict);
@ -435,11 +533,34 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
void* const cdictBuffer = malloc(cdictSize);
if (cdictBuffer==NULL) goto _output_error;
{ ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
dictBuffer, dictSize,
0 /* by Reference */, ZSTD_dm_auto,
cParams);
if (cdict == NULL) {
DISPLAY("ZSTD_initStaticCDict failed ");
goto _output_error;
}
cSize = ZSTD_compress_usingCDict(cctx,
compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict);
if (ZSTD_isError(cSize)) {
DISPLAY("ZSTD_compress_usingCDict failed ");
goto _output_error;
} }
free(cdictBuffer);
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem const customMem = { NULL, NULL, NULL };
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict, fParams);
ZSTD_freeCDict(cdict);
@ -488,6 +609,22 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error;
ZSTD_freeCDict(cdict);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error;
ZSTD_freeCDict(cdict);
}
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
free(dictBuffer);
free(samplesSizes);
@ -566,9 +703,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const wrongSrcSize = (srcSize + 1000);
ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
params.fParams.contentSizeFlag = 1;
{ size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize);
if (ZSTD_isError(result)) goto _output_error;
}
CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
{ size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
if (!ZSTD_isError(result)) goto _output_error;
if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@ -586,6 +721,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, 5) );
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
@ -749,8 +885,23 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
}
#undef CHECK
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
#define CHECK(cond, ...) { \
if (cond) { \
DISPLAY("Error => "); \
DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
goto _output_error; \
} }
#define CHECK_Z(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
DISPLAY("Error => %s : %s ", \
#f, ZSTD_getErrorName(err)); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
goto _output_error; \
} }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
{
@ -863,8 +1014,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* frame header decompression test */
{ ZSTD_frameHeader dParams;
size_t const check = ZSTD_getFrameHeader(&dParams, cBuffer, cSize);
CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
CHECK_Z( ZSTD_getFrameHeader(&dParams, cBuffer, cSize) );
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
}
@ -951,20 +1101,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
if (FUZ_rand(&lseed) & 0xF) {
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
} }
CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
}
ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
@ -996,8 +1143,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize) {

2
tests/symbols.c
View File

@ -95,7 +95,7 @@ static const void *symbols[] = {
&ZSTD_nextSrcSizeToDecompress,
&ZSTD_decompressContinue,
&ZSTD_nextInputType,
&ZSTD_getBlockSizeMax,
&ZSTD_getBlockSize,
&ZSTD_compressBlock,
&ZSTD_decompressBlock,
&ZSTD_insertBlock,

900
tests/zstreamtest.c
View File

File diff suppressed because it is too large Load Diff

15
zlibWrapper/Makefile
View File

@ -18,9 +18,12 @@ EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wstrict-aliasing=1
# Define *.exe as extension for Windows systems
@ -48,8 +51,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
#cp example$(EXT).gz example$(EXT)_gz.gz
./minigzip_zstd -d example$(EXT).gz
@echo ---- minigzip end ----
./zwrapbench -qb3B1K $(TEST_FILE)
./zwrapbench -rqb1e5 ../lib ../programs ../tests
./zwrapbench -qi1b3B1K $(TEST_FILE)
./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
@ -61,8 +64,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
$(VALGRIND) ./fitblk 40960 <$(TEST_FILE)
$(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE)
$(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE)
$(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE)
$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
$(VALGRIND) ./zwrapbench -qi1b3B1K $(TEST_FILE)
$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
#.c.o:
# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<

4
zlibWrapper/examples/zwrapbench.c
View File

@ -90,7 +90,7 @@ static clock_t g_time = 0;
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
@ -236,7 +236,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (compressor == BMK_ZSTD) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1 /*byRef*/, ZSTD_dm_auto, zparams.cParams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
do {

625
zlibWrapper/zstd_zlibwrapper.c
View File

@ -8,35 +8,42 @@
*/
/* === Tuning parameters === */
#ifndef ZWRAP_USE_ZSTD
#define ZWRAP_USE_ZSTD 0
#endif
/* === Dependencies === */
#include <stdlib.h>
#include <stdio.h> /* vsprintf */
#include <stdarg.h> /* va_list, for z_gzprintf */
#define NO_DUMMY_DECL
#define ZLIB_CONST
#include <zlib.h> /* without #define Z_PREFIX */
#include "zstd_zlibwrapper.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER */
#include "zstd.h"
#include "zstd_internal.h" /* defaultCustomMem */
#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */
/* === Constants === */
#define Z_INFLATE_SYNC 8
#define ZLIB_HEADERSIZE 4
#define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min
#define ZWRAP_DEFAULT_CLEVEL 3 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
#define LOG_WRAPPERC(...) /* printf(__VA_ARGS__) */
#define LOG_WRAPPERD(...) /* printf(__VA_ARGS__) */
/* === Debug === */
#define LOG_WRAPPERC(...) /* fprintf(stderr, __VA_ARGS__) */
#define LOG_WRAPPERD(...) /* fprintf(stderr, __VA_ARGS__) */
#define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; }
#define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; }
#ifndef ZWRAP_USE_ZSTD
#define ZWRAP_USE_ZSTD 0
#endif
static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
/* === Wrapper === */
static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
void ZWRAP_useZSTDcompression(int turn_on) { g_ZWRAP_useZSTDcompression = turn_on; }
@ -62,7 +69,7 @@ static void* ZWRAP_allocFunction(void* opaque, size_t size)
{
z_streamp strm = (z_streamp) opaque;
void* address = strm->zalloc(strm->opaque, 1, (uInt)size);
/* printf("ZWRAP alloc %p, %d \n", address, (int)size); */
/* LOG_WRAPPERC("ZWRAP alloc %p, %d \n", address, (int)size); */
return address;
}
@ -70,12 +77,12 @@ static void ZWRAP_freeFunction(void* opaque, void* address)
{
z_streamp strm = (z_streamp) opaque;
strm->zfree(strm->opaque, address);
/* if (address) printf("ZWRAP free %p \n", address); */
/* if (address) LOG_WRAPPERC("ZWRAP free %p \n", address); */
}
/* *** Compression *** */
/* === Compression === */
typedef enum { ZWRAP_useInit, ZWRAP_useReset, ZWRAP_streamEnd } ZWRAP_state_t;
typedef struct {
@ -95,16 +102,16 @@ typedef ZWRAP_CCtx internal_state;
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
static size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
{
if (zwc==NULL) return 0; /* support free on NULL */
if (zwc->zbc) ZSTD_freeCStream(zwc->zbc);
zwc->customMem.customFree(zwc->customMem.opaque, zwc);
ZSTD_freeCStream(zwc->zbc);
ZSTD_free(zwc, zwc->customMem);
return 0;
}
ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
{
ZWRAP_CCtx* zwc;
@ -113,37 +120,36 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
}
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
zwc->customMem = ZWRAP_customMem; }
} else {
zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx));
zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
if (zwc==NULL) return NULL;
memset(zwc, 0, sizeof(ZWRAP_CCtx));
memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
return zwc;
}
int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize)
static int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize)
{
LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc);
if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR;
if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
{ ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
size_t errorCode;
LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
errorCode = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; }
{ ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
size_t initErr;
LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n",
(int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(initErr)) return Z_STREAM_ERROR;
}
return Z_OK;
}
int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
static int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
{
LOG_WRAPPERC("- ZWRAPC_finishWithError=%d\n", error);
if (zwc) ZWRAP_freeCCtx(zwc);
@ -152,7 +158,7 @@ int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
}
int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
static int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
{
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
strm->msg = message;
@ -219,7 +225,7 @@ int ZWRAP_deflateReset_keepDict(z_streamp strm)
return deflateReset(strm);
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
if (zwc) {
if (zwc) {
zwc->streamEnd = 0;
zwc->totalInBytes = 0;
}
@ -277,34 +283,36 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
ZWRAP_CCtx* zwc;
if (!g_ZWRAP_useZSTDcompression) {
int res;
LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
res = deflate(strm, flush);
return res;
LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
return deflate(strm, flush);
}
zwc = (ZWRAP_CCtx*) strm->state;
if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; }
if (zwc->zbc == NULL) {
int res;
zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
{ int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
if (initErr != Z_OK) return ZWRAPC_finishWithError(zwc, strm, initErr); }
if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
} else {
if (zwc->totalInBytes == 0) {
if (zwc->comprState == ZWRAP_useReset) {
size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); }
size_t const resetErr = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
if (ZSTD_isError(resetErr)) {
LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n",
ZSTD_getErrorName(resetErr));
return ZWRAPC_finishWithError(zwc, strm, 0);
}
} else {
int res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
}
}
}
} /* (zwc->totalInBytes == 0) */
} /* ! (zwc->zbc == NULL) */
LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (strm->avail_in > 0) {
@ -314,9 +322,9 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
zwc->outBuffer.dst = strm->next_out;
zwc->outBuffer.size = strm->avail_out;
zwc->outBuffer.pos = 0;
{ size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer);
{ size_t const cErr = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer);
LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size);
if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0);
if (ZSTD_isError(cErr)) return ZWRAPC_finishWithError(zwc, strm, 0);
}
strm->next_out += zwc->outBuffer.pos;
strm->total_out += zwc->outBuffer.pos;
@ -346,8 +354,12 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
strm->next_out += zwc->outBuffer.pos;
strm->total_out += zwc->outBuffer.pos;
strm->avail_out -= zwc->outBuffer.pos;
if (bytesLeft == 0) { zwc->streamEnd = 1; LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; }
}
if (bytesLeft == 0) {
zwc->streamEnd = 1;
LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n",
(int)strm->total_in, (int)strm->avail_out, (int)strm->total_out);
return Z_STREAM_END;
} }
else
if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) {
size_t bytesLeft;
@ -410,12 +422,13 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
/* *** Decompression *** */
/* === Decompression === */
typedef enum { ZWRAP_ZLIB_STREAM, ZWRAP_ZSTD_STREAM, ZWRAP_UNKNOWN_STREAM } ZWRAP_stream_type;
typedef struct {
ZSTD_DStream* zbd;
char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
char headerBuf[16]; /* must be >= ZSTD_frameHeaderSize_min */
int errorCount;
unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */
ZWRAP_state_t decompState;
@ -427,10 +440,48 @@ typedef struct {
char *version;
int windowBits;
ZSTD_customMem customMem;
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
z_stream allocFunc; /* just to copy zalloc, zfree, opaque */
} ZWRAP_DCtx;
static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
{
zwd->errorCount = 0;
zwd->outBuffer.pos = 0;
zwd->outBuffer.size = 0;
}
static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{
ZWRAP_DCtx* zwd;
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN); /* check static buffer size condition */
if (strm->zalloc && strm->zfree) {
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx));
zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */
{ ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
zwd->customMem = ZWRAP_customMem; }
} else {
zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
if (zwd==NULL) return NULL;
}
ZWRAP_initDCtx(zwd);
return zwd;
}
static size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
{
if (zwd==NULL) return 0; /* support free on null */
ZSTD_freeDStream(zwd->zbd);
ZSTD_free(zwd->version, zwd->customMem);
ZSTD_free(zwd, zwd->customMem);
return 0;
}
int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
{
if (strm == NULL) return 0;
@ -438,61 +489,17 @@ int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
}
void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
{
zwd->errorCount = 0;
zwd->outBuffer.pos = 0;
zwd->outBuffer.size = 0;
}
ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{
ZWRAP_DCtx* zwd;
if (strm->zalloc && strm->zfree) {
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx));
memcpy(&zwd->allocFunc, strm, sizeof(z_stream));
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
}
} else {
zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx));
if (zwd==NULL) return NULL;
memset(zwd, 0, sizeof(ZWRAP_DCtx));
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN); /* if compilation fails here, assertion is false */
ZWRAP_initDCtx(zwd);
return zwd;
}
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
{
if (zwd==NULL) return 0; /* support free on null */
if (zwd->zbd) ZSTD_freeDStream(zwd->zbd);
if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
zwd->customMem.customFree(zwd->customMem.opaque, zwd);
return 0;
}
int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error)
static int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error)
{
LOG_WRAPPERD("- ZWRAPD_finishWithError=%d\n", error);
if (zwd) ZWRAP_freeDCtx(zwd);
if (strm) strm->state = NULL;
ZWRAP_freeDCtx(zwd);
strm->state = NULL;
return (error) ? error : Z_STREAM_ERROR;
}
int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
{
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
strm->msg = message;
if (zwd == NULL) return Z_STREAM_ERROR;
@ -504,26 +511,25 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
const char *version, int stream_size))
{
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
strm->reserved = ZWRAP_ZLIB_STREAM;
return inflateInit(strm);
}
{
ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
LOG_WRAPPERD("- inflateInit\n");
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
{ ZWRAP_DCtx* const zwd = ZWRAP_createDCtx(strm);
LOG_WRAPPERD("- inflateInit\n");
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version);
zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem);
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version);
zwd->stream_size = stream_size;
zwd->totalInBytes = 0;
strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
strm->total_in = 0;
strm->total_out = 0;
strm->reserved = ZWRAP_UNKNOWN_STREAM; /* mark as unknown steam */
strm->adler = 0;
zwd->stream_size = stream_size;
zwd->totalInBytes = 0;
strm->state = (struct internal_state*) zwd;
strm->total_in = 0;
strm->total_out = 0;
strm->reserved = ZWRAP_UNKNOWN_STREAM;
strm->adler = 0;
}
return Z_OK;
@ -537,15 +543,14 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits,
return inflateInit2_(strm, windowBits, version, stream_size);
}
{
int ret = z_inflateInit_ (strm, version, stream_size);
LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
if (ret == Z_OK) {
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd->windowBits = windowBits;
}
return ret;
{ int const ret = z_inflateInit_ (strm, version, stream_size);
LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
if (ret == Z_OK) {
ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd->windowBits = windowBits;
}
return ret;
}
}
@ -555,7 +560,7 @@ int ZWRAP_inflateReset_keepDict(z_streamp strm)
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateReset(strm);
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
{ ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
ZWRAP_initDCtx(zwd);
zwd->decompState = ZWRAP_useReset;
@ -574,10 +579,10 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateReset(strm);
{ int ret = ZWRAP_inflateReset_keepDict(strm);
{ int const ret = ZWRAP_inflateReset_keepDict(strm);
if (ret != Z_OK) return ret; }
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
{ ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd->decompState = ZWRAP_useInit; }
@ -592,9 +597,9 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateReset2(strm, windowBits);
{ int ret = z_inflateReset (strm);
{ int const ret = z_inflateReset (strm);
if (ret == Z_OK) {
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd->windowBits = windowBits;
}
@ -612,11 +617,10 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateSetDictionary(strm, dictionary, dictLength);
{ size_t errorCode;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
{ ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
{ size_t const initErr = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
if (ZSTD_isError(initErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
zwd->decompState = ZWRAP_useReset;
if (zwd->totalInBytes == ZSTD_HEADERSIZE) {
@ -626,14 +630,14 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = 0;
zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode));
return ZWRAPD_finishWithError(zwd, strm, 0);
}
}
}
{ size_t const errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n",
(int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n",
ZSTD_getErrorName(errorCode));
return ZWRAPD_finishWithError(zwd, strm, 0);
} } } }
return Z_OK;
}
@ -642,156 +646,175 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
{
ZWRAP_DCtx* zwd;
int res;
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
int const result = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, result);
return result;
}
if (strm->avail_in <= 0) return Z_OK;
{ size_t errorCode, srcSize;
zwd = (ZWRAP_DCtx*) strm->state;
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
zwd = (ZWRAP_DCtx*) strm->state;
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (zwd == NULL) return Z_STREAM_ERROR;
if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
if (zwd == NULL) return Z_STREAM_ERROR;
if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
if (zwd->windowBits)
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
else
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
}
} else {
srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes);
memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
strm->total_in += srcSize;
zwd->totalInBytes += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2;
strm2.next_in = strm->next_in;
strm2.avail_in = strm->avail_in;
strm2.next_out = strm->next_out;
strm2.avail_out = strm->avail_out;
if (zwd->windowBits)
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
else
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
/* inflate header */
strm->next_in = (unsigned char*)zwd->headerBuf;
strm->avail_in = ZLIB_HEADERSIZE;
strm->avail_out = 0;
errorCode = inflate(strm, Z_NO_FLUSH);
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (strm->avail_in > 0) goto error;
strm->next_in = strm2.next_in;
strm->avail_in = strm2.avail_in;
strm->next_out = strm2.next_out;
strm->avail_out = strm2.avail_out;
strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
}
}
}
strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
if (!zwd->zbd) {
zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
zwd->decompState = ZWRAP_useInit;
}
if (zwd->totalInBytes < ZSTD_HEADERSIZE)
{
if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
if (zwd->decompState == ZWRAP_useInit) {
errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
} else {
errorCode = ZSTD_resetDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) goto error;
}
} else {
srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes);
memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
strm->total_in += srcSize;
zwd->totalInBytes += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK;
if (zwd->decompState == ZWRAP_useInit) {
errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
} else {
errorCode = ZSTD_resetDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) goto error;
if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
{ int const initErr = (zwd->windowBits) ?
inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) :
inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr);
if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr);
}
zwd->inBuffer.src = zwd->headerBuf;
zwd->inBuffer.size = ZSTD_HEADERSIZE;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = 0;
zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
strm->reserved = ZWRAP_ZLIB_STREAM;
{ size_t const freeErr = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(freeErr)) goto error; }
{ int const result = (flush == Z_INFLATE_SYNC) ?
inflateSync(strm) :
inflate(strm, flush);
LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return result;
} }
} else { /* ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */
size_t const srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes);
memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
strm->total_in += srcSize;
zwd->totalInBytes += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2;
strm2.next_in = strm->next_in;
strm2.avail_in = strm->avail_in;
strm2.next_out = strm->next_out;
strm2.avail_out = strm->avail_out;
{ int const initErr = (zwd->windowBits) ?
inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) :
inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr);
if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr);
}
/* inflate header */
strm->next_in = (unsigned char*)zwd->headerBuf;
strm->avail_in = ZLIB_HEADERSIZE;
strm->avail_out = 0;
{ int const dErr = inflate(strm, Z_NO_FLUSH);
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n",
dErr, (int)strm->avail_in);
if (dErr != Z_OK)
return ZWRAPD_finishWithError(zwd, strm, dErr);
}
if (strm->avail_in > 0) goto error;
strm->next_in = strm2.next_in;
strm->avail_in = strm2.avail_in;
strm->next_out = strm2.next_out;
strm->avail_out = strm2.avail_out;
strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
{ size_t const freeErr = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(freeErr)) goto error; }
{ int const result = (flush == Z_INFLATE_SYNC) ?
inflateSync(strm) :
inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return result;
} } } /* if ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */
} /* (zwd->totalInBytes < ZLIB_HEADERSIZE) */
strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
if (!zwd->zbd) {
zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
zwd->decompState = ZWRAP_useInit;
}
if (zwd->totalInBytes < ZSTD_HEADERSIZE) {
if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
if (zwd->decompState == ZWRAP_useInit) {
size_t const initErr = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(initErr)) {
LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n",
ZSTD_getErrorName(initErr));
goto error;
}
if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */
} else {
size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
if (ZSTD_isError(resetErr)) goto error;
}
}
} else {
size_t const srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes);
memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
strm->total_in += srcSize;
zwd->totalInBytes += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK;
zwd->inBuffer.src = strm->next_in;
zwd->inBuffer.size = strm->avail_in;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = strm->avail_out;
zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out);
if (ZSTD_isError(errorCode)) {
if (zwd->decompState == ZWRAP_useInit) {
size_t const initErr = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(initErr)) {
LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n",
ZSTD_getErrorName(initErr));
goto error;
}
} else {
size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
if (ZSTD_isError(resetErr)) goto error;
}
zwd->inBuffer.src = zwd->headerBuf;
zwd->inBuffer.size = ZSTD_HEADERSIZE;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = 0;
zwd->outBuffer.pos = 0;
{ size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n",
(int)dErr, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (ZSTD_isError(dErr)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(dErr));
goto error;
} }
if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */
}
} /* (zwd->totalInBytes < ZSTD_HEADERSIZE) */
zwd->inBuffer.src = strm->next_in;
zwd->inBuffer.size = strm->avail_in;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = strm->avail_out;
zwd->outBuffer.pos = 0;
{ size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n",
(int)dErr, (int)strm->avail_in, (int)strm->avail_out);
if (ZSTD_isError(dErr)) {
zwd->errorCount++;
LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount);
LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n",
ZSTD_getErrorName(dErr), zwd->errorCount);
if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
}
LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n",
(int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
strm->next_out += zwd->outBuffer.pos;
strm->total_out += zwd->outBuffer.pos;
strm->avail_out -= zwd->outBuffer.pos;
@ -799,13 +822,16 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
zwd->totalInBytes += zwd->inBuffer.pos;
strm->next_in += zwd->inBuffer.pos;
strm->avail_in -= zwd->inBuffer.pos;
if (errorCode == 0) {
LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (dErr == 0) {
LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
(int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
zwd->decompState = ZWRAP_streamEnd;
return Z_STREAM_END;
}
}
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
} /* dErr lifetime */
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
(int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
return Z_OK;
error:
@ -818,13 +844,13 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateEnd(strm);
LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
{ size_t errorCode;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n",
(int)(strm->total_in), (int)(strm->total_out));
{ ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_OK; /* structures are already freed */
{ size_t const freeErr = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(freeErr)) return Z_STREAM_ERROR; }
strm->state = NULL;
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
}
return Z_OK;
}
@ -841,8 +867,6 @@ ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm))
/* Advanced compression functions */
ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
z_streamp source))
@ -982,7 +1006,7 @@ ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags();
/* utility functions */
/* === utility functions === */
#ifndef Z_SOLO
ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
@ -991,11 +1015,14 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
if (!g_ZWRAP_useZSTDcompression)
return compress(dest, destLen, source, sourceLen);
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
{ size_t dstCapacity = *destLen;
size_t const cSize = ZSTD_compress(dest, dstCapacity,
source, sourceLen,
ZWRAP_DEFAULT_CLEVEL);
LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n",
(int)sourceLen, (int)dstCapacity);
if (ZSTD_isError(cSize)) return Z_STREAM_ERROR;
*destLen = cSize;
}
return Z_OK;
}
@ -1009,9 +1036,9 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen,
return compress2(dest, destLen, source, sourceLen, level);
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
size_t const cSize = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
if (ZSTD_isError(cSize)) return Z_STREAM_ERROR;
*destLen = cSize;
}
return Z_OK;
}
@ -1029,13 +1056,13 @@ ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen))
ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen,
const Bytef *source, uLong sourceLen))
{
if (sourceLen < 4 || MEM_readLE32(source) != ZSTD_MAGICNUMBER)
if (!ZSTD_isFrame(source, sourceLen))
return uncompress(dest, destLen, source, sourceLen);
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
size_t const dSize = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
if (ZSTD_isError(dSize)) return Z_STREAM_ERROR;
*destLen = dSize;
}
return Z_OK;
}