facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move posInSequence correction to correct location

Browse Source
This commit is contained in:
senhuang42 2020-10-06 16:16:41 -04:00
parent 0c515590d8
commit abce708a56
3 changed files with 83 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
doc/zstd_manual.html
View File

@ -1,10 +1,10 @@
<html> <html>
<head> <head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.4.5 Manual</title> <title>zstd 1.4.6 Manual</title>
</head> </head>
<body> <body>
<h1>zstd 1.4.5 Manual</h1> <h1>zstd 1.4.6 Manual</h1>
<hr> <hr>
<a name="Contents"></a><h2>Contents</h2> <a name="Contents"></a><h2>Contents</h2>
<ol> <ol>
@ -24,19 +24,20 @@
<li><a href="#Chapter14">experimental API (static linking only)</a></li> <li><a href="#Chapter14">experimental API (static linking only)</a></li>
<li><a href="#Chapter15">Frame size functions</a></li> <li><a href="#Chapter15">Frame size functions</a></li>
<li><a href="#Chapter16">Memory management</a></li> <li><a href="#Chapter16">Memory management</a></li>
<li><a href="#Chapter17">Advanced compression functions</a></li> <li><a href="#Chapter17">This API is temporary and is expected to change or disappear in the future!</a></li>
<li><a href="#Chapter18">Advanced decompression functions</a></li> <li><a href="#Chapter18">Advanced compression functions</a></li>
<li><a href="#Chapter19">Advanced streaming functions</a></li> <li><a href="#Chapter19">Advanced decompression functions</a></li>
<li><a href="#Chapter20">! ZSTD_initCStream_usingDict() :</a></li> <li><a href="#Chapter20">Advanced streaming functions</a></li>
<li><a href="#Chapter21">! ZSTD_initCStream_advanced() :</a></li> <li><a href="#Chapter21">! ZSTD_initCStream_usingDict() :</a></li>
<li><a href="#Chapter22">! ZSTD_initCStream_usingCDict() :</a></li> <li><a href="#Chapter22">! ZSTD_initCStream_advanced() :</a></li>
<li><a href="#Chapter23">! ZSTD_initCStream_usingCDict_advanced() :</a></li> <li><a href="#Chapter23">! ZSTD_initCStream_usingCDict() :</a></li>
<li><a href="#Chapter24">This function is deprecated, and is equivalent to:</a></li> <li><a href="#Chapter24">! ZSTD_initCStream_usingCDict_advanced() :</a></li>
<li><a href="#Chapter25">This function is deprecated, and is equivalent to:</a></li> <li><a href="#Chapter25">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter26">Buffer-less and synchronous inner streaming functions</a></li> <li><a href="#Chapter26">This function is deprecated, and is equivalent to:</a></li>
<li><a href="#Chapter27">Buffer-less streaming compression (synchronous mode)</a></li> <li><a href="#Chapter27">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter28">Buffer-less streaming decompression (synchronous mode)</a></li> <li><a href="#Chapter28">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter29">Block level API</a></li> <li><a href="#Chapter29">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter30">Block level API</a></li>
</ol> </ol>
<hr> <hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre> <a name="Chapter1"></a><h2>Introduction</h2><pre>
@ -72,8 +73,14 @@
<a name="Chapter2"></a><h2>Version</h2><pre></pre> <a name="Chapter2"></a><h2>Version</h2><pre></pre>
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b> <pre><b>unsigned ZSTD_versionNumber(void);
</b></pre><BR> </b><p> Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE).
</p></pre><BR>
<pre><b>const char* ZSTD_versionString(void);
</b><p> Return runtime library version, like "1.4.5". Requires v1.3.0+.
</p></pre><BR>
<a name="Chapter3"></a><h2>Simple API</h2><pre></pre> <a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
@ -308,16 +315,20 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b> ZSTD_c_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
</b>/* multi-threading parameters */<b> </b>/* multi-threading parameters */<b>
</b>/* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b> </b>/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).<b>
* They return an error otherwise. */ * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
* In a situation where it's unknown if the linked library supports multi-threading or not,
* setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
*/
ZSTD_c_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b> ZSTD_c_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b>
* When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads. * while compression is performed in parallel, within worker thread(s).
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
* More workers improve speed, but also increase memory usage. * More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
* compression is performed inside Caller's thread, and all invocations are blocking */
ZSTD_c_jobSize=401, </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b> ZSTD_c_jobSize=401, </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b>
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters. * 0 means default, which is dynamically determined based on compression parameters.
@ -346,6 +357,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* ZSTD_c_literalCompressionMode * ZSTD_c_literalCompressionMode
* ZSTD_c_targetCBlockSize * ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint * ZSTD_c_srcSizeHint
* ZSTD_c_enableDedicatedDictSearch
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly; * note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change. * also, the enums values themselves are unstable and can still change.
@ -356,7 +368,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003, ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004 ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam8=1005
} ZSTD_cParameter; } ZSTD_cParameter;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
@ -456,11 +469,13 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* At the time of this writing, they include : * At the time of this writing, they include :
* ZSTD_d_format * ZSTD_d_format
* ZSTD_d_stableOutBuffer * ZSTD_d_stableOutBuffer
* ZSTD_d_forceIgnoreChecksum
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly * note : never ever use experimentalParam? names directly
*/ */
ZSTD_d_experimentalParam1=1000, ZSTD_d_experimentalParam1=1000,
ZSTD_d_experimentalParam2=1001 ZSTD_d_experimentalParam2=1001,
ZSTD_d_experimentalParam3=1002
} ZSTD_dParameter; } ZSTD_dParameter;
</b></pre><BR> </b></pre><BR>
@ -591,8 +606,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
- Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- output->pos must be <= dstCapacity, input->pos must be <= srcSize - output->pos must be <= dstCapacity, input->pos must be <= srcSize
- output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- endOp must be a valid directive
- When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
- When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
and then immediately returns, just indicating that there is some data remaining to be flushed. and then immediately returns, just indicating that there is some data remaining to be flushed.
The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@ -951,6 +967,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Decoder cannot recognise automatically this format, requiring this instruction. */ * Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e; } ZSTD_format_e;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */<b>
ZSTD_d_validateChecksum = 0,
ZSTD_d_ignoreChecksum = 1
} ZSTD_forceIgnoreChecksum_e;
</b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
</b>/* Note: this enum and the behavior it controls are effectively internal<b> </b>/* Note: this enum and the behavior it controls are effectively internal<b>
* implementation details of the compressor. They are expected to continue * implementation details of the compressor. They are expected to continue
@ -1148,7 +1170,10 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</p></pre><BR> </p></pre><BR>
<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre> <a name="Chapter17"></a><h2>This API is temporary and is expected to change or disappear in the future!</h2><pre>
<BR></pre>
<a name="Chapter18"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression </b><p> Create a digested dictionary for compression
@ -1264,8 +1289,10 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
<pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); <pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
</b><p> Similar to ZSTD_CCtx_setParameter. </b><p> Similar to ZSTD_CCtx_setParameter.
Set one compression parameter, selected by enum ZSTD_cParameter. Set one compression parameter, selected by enum ZSTD_cParameter.
Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). Parameters must be applied to a ZSTD_CCtx using
@result : 0, or an error code (which can be tested with ZSTD_isError()). ZSTD_CCtx_setParametersUsingCCtxParams().
@result : a code representing success or failure (which can be tested with
ZSTD_isError()).
</p></pre><BR> </p></pre><BR>
@ -1298,7 +1325,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR> </p></pre><BR>
<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre> <a name="Chapter19"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@ -1360,7 +1387,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR> </p></pre><BR>
<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. <a name="Chapter20"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
Once Advanced API reaches "stable" status, Once Advanced API reaches "stable" status,
redundant functions will be deprecated, and then at some point removed. redundant functions will be deprecated, and then at some point removed.
<BR></pre> <BR></pre>
@ -1382,7 +1409,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel, int compressionLevel,
unsigned long long pledgedSrcSize); unsigned long long pledgedSrcSize);
</pre></b><BR> </pre></b><BR>
<a name="Chapter20"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to: <a name="Chapter21"></a><h2>! ZSTD_initCStream_usingDict() :</h2><pre> This function is deprecated, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@ -1395,7 +1422,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre> <BR></pre>
<a name="Chapter21"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to: <a name="Chapter22"></a><h2>! ZSTD_initCStream_advanced() :</h2><pre> This function is deprecated, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd parameter and leave the rest as-is. // Pseudocode: Set each zstd parameter and leave the rest as-is.
for ((param, value) : params) { for ((param, value) : params) {
@ -1411,7 +1438,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre> <BR></pre>
<a name="Chapter22"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to: <a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict() :</h2><pre> This function is deprecated, and equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_refCDict(zcs, cdict); ZSTD_CCtx_refCDict(zcs, cdict);
@ -1420,7 +1447,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
<BR></pre> <BR></pre>
<a name="Chapter23"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to: <a name="Chapter24"></a><h2>! ZSTD_initCStream_usingCDict_advanced() :</h2><pre> This function is DEPRECATED, and is approximately equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd frame parameter and leave the rest as-is. // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
for ((fParam, value) : fParams) { for ((fParam, value) : fParams) {
@ -1488,7 +1515,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
*/ */
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
</pre></b><BR> </pre></b><BR>
<a name="Chapter24"></a><h2>This function is deprecated, and is equivalent to:</h2><pre> <a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
ZSTD_DCtx_refDDict(zds, ddict); ZSTD_DCtx_refDDict(zds, ddict);
@ -1497,7 +1524,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
<BR></pre> <BR></pre>
<a name="Chapter25"></a><h2>This function is deprecated, and is equivalent to:</h2><pre> <a name="Chapter26"></a><h2>This function is deprecated, and is equivalent to:</h2><pre>
ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
re-use decompression parameters from previous init; saves dictionary loading re-use decompression parameters from previous init; saves dictionary loading
@ -1505,14 +1532,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
<BR></pre> <BR></pre>
<a name="Chapter26"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> <a name="Chapter27"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory. This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below. But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience. Prefer normal streaming API for an easier experience.
<BR></pre> <BR></pre>
<a name="Chapter27"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> <a name="Chapter28"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations. A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations. ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@ -1548,7 +1575,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter28"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> <a name="Chapter29"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. A ZSTD_DCtx object can be re-used multiple times.
@ -1644,7 +1671,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter29"></a><h2>Block level API</h2><pre></pre> <a name="Chapter30"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). <pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.

12
lib/compress/zstd_ldm.c
View File

@ -559,20 +559,22 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
* Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
*/ */
static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { static void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = rawSeqStore->posInSequence + nbBytes; U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) { while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
if (currPos >= currSeq.litLength + currSeq.matchLength) { if (currPos >= currSeq.litLength + currSeq.matchLength) {
currPos -= currSeq.litLength + currSeq.matchLength; currPos -= currSeq.litLength + currSeq.matchLength;
rawSeqStore->pos++; rawSeqStore->pos++;
if (currPos == 0) {
rawSeqStore->posInSequence = 0;
}
} else { } else {
rawSeqStore->posInSequence = currPos; rawSeqStore->posInSequence = currPos;
break; break;
} }
} }
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
rawSeqStore->posInSequence = 0;
}
assert(rawSeqStore->posInSequence <=
rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
} }
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
@ -597,7 +599,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize);
if (rawSeqStore->pos >= rawSeqStore->size) { if (rawSeqStore->pos >= rawSeqStore->size) {
/* If we're done with rawSeqStore, invalidate the one in matchState as well */ /* If we're done with rawSeqStore, invalidate the one in matchState as well */
ms->ldmSeqStore.pos = rawSeqStore->size; ms->ldmSeqStore.size = 0;
} }
return lastLLSize; return lastLLSize;
} }

10
lib/compress/zstd_opt.c
View File

@ -780,20 +780,22 @@ typedef struct {
* Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'. * Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'.
*/ */
static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = rawSeqStore->posInSequence + nbBytes; U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) { while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
if (currPos >= currSeq.litLength + currSeq.matchLength) { if (currPos >= currSeq.litLength + currSeq.matchLength) {
currPos -= currSeq.litLength + currSeq.matchLength; currPos -= currSeq.litLength + currSeq.matchLength;
rawSeqStore->pos++; rawSeqStore->pos++;
if (currPos == 0) {
rawSeqStore->posInSequence = 0;
}
} else { } else {
rawSeqStore->posInSequence = currPos; rawSeqStore->posInSequence = currPos;
break; break;
} }
} }
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
rawSeqStore->posInSequence = 0;
}
assert(rawSeqStore->posInSequence <=
rawSeqStore->seq[rawSeqStore->pos].litLength + rawSeqStore->seq[rawSeqStore->pos].matchLength);
} }
/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): /* ZSTD_opt_getNextMatchAndUpdateSeqStore():