From 34e146f5482af7105d2f3539a3f2afb8b6c80c36 Mon Sep 17 00:00:00 2001
From: Yann Collet Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- - The function is always blocking.
+ - The function is always blocking, returns when compression is completed.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
@return : compressed size written into `dst` (<= `dstCapacity),
or an error code if it fails (which can be tested using ZSTD_isError()).
@@ -648,24 +654,25 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_e_flush=1, /* flush any data provided so far,
* it creates (at least) one new block, that can be decoded immediately on reception;
* frame will continue: any future data can still reference previously compressed data, improving compression. */
- ZSTD_e_end=2 /* flush any remaining data and close current frame.
- * any additional data starts a new frame.
- * each frame is independent (does not reference any content from previous frame). */
+ ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
+ * note that frame is only closed after compressed data is fully flushed (return value == 0).
+ * After that point, any additional data starts a new frame.
+ * note : each frame is independent (does not reference any content from previous frame). */
} ZSTD_EndDirective;
Behave about the same as ZSTD_compressStream, with additional control on end directive.
+ Behaves about the same as ZSTD_compressStream, with additional control on end directive.
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- - In single-thread mode (default), function is blocking : it completes its job before returning to caller.
- - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
- and then immediately returns, just indicating that there is some data remaining to be flushed.
- The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
+ - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
+ - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads,
+ and then immediately returns, just indicating that there is some data remaining to be flushed.
+ The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- Exception : if the first call requests a ZSTD_e_end directive, the function delegates to ZSTD_compress2() which is always blocking.
- @return provides a minimum amount of data remaining to be flushed from internal buffers
or an error code, which can be tested using ZSTD_isError().
@@ -679,12 +686,40 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
- This protects a decoder context from reserving too much memory for itself (potential attack scenario).
- This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
- By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
- @return : 0, or an error code (which can be tested using ZSTD_isError()).
+ All parameters must belong to an interval with lower and upper bounds,
+ otherwise they will either trigger an error or be automatically clamped.
+ @return : a structure, ZSTD_bounds, which contains
+ - an error status field, which must be tested using ZSTD_isError()
+ - both lower and upper bounds, inclusive
+
+ Set one compression parameter, selected by enum ZSTD_dParameter.
+ All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
+ Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
+ Setting a parameter is only possible during frame initialization (before starting decompression).
+ @return : an error code (which can be tested using ZSTD_isError()).
typedef enum {
@@ -512,7 +518,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* ZSTD_p_forceMaxWindow
* ZSTD_p_forceAttachDict
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
- * note : never use experimentalParam names directly
+ * note : never ever use experimentalParam? names directly
*/
ZSTD_p_experimentalParam1=500,
ZSTD_p_experimentalParam2=10,
@@ -636,7 +642,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
const void* src, size_t srcSize);
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
-
-size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
-
typedef enum {
+
+ ZSTD_d_windowLogMax=100, /* Used to select a limit beyond which
+ * the streaming API will refuse to allocate memory buffer
+ * in order to protect the host from unreasonable memory requirements.
+ * This parameter is only useful in streaming mode, as no internal buffer is allocated in single-pass mode.
+ * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */
+
+ /* note : additional experimental parameters are also available
+ * within the experimental section of the API.
+ * At the time of this writing, they include :
+ * ZSTD_p_format
+ * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+ * note : never ever use experimentalParam? names directly
+ */
+ ZSTD_d_experimentalParam1=1000
+
+} ZSTD_dParameter;
+
+ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); /* not implemented yet */
+
+
+size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); /* not implemented yet */
+
@@ -711,7 +746,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : Currently, only one dictionary can be managed.
Referencing a new dictionary effectively "discards" any previous one.
- Special : adding a NULL DDict means "return to no-dictionary mode".
+ Special: referencing a NULL DDict means "return to no-dictionary mode".
Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input); -Behave the same as ZSTD_decompressStream. - Decompression parameters cannot be changed once decompression is started. - @return : an error code, which can be tested using ZSTD_isError() - if >0, a hint, nb of expected input bytes for next invocation. - `0` means : a frame has just been fully decoded and flushed. - -
The following symbols and constants are not planned to join "stable API" status in the near future. @@ -1152,6 +1176,15 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?)
size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); +Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + This protects a decoder context from reserving too much memory for itself (potential attack scenario). + This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + @return : 0, or an error code (which can be tested using ZSTD_isError()). + +
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);Instruct the decoder context about what kind of data to decode next. This instruction is mandatory to decode data without a fully-formed header, @@ -1159,11 +1192,11 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); @return : 0, or an error code (which can be tested using ZSTD_isError()).
size_t ZSTD_decompress_generic_simpleArgs ( +size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos); -Same as ZSTD_decompress_generic(), +
Same as ZSTD_decompressStream(), but using only integral types as arguments. This can be helpful for binders from dynamic languages which have troubles handling structures containing memory pointers. diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9f269dde..68acebc6 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -42,7 +42,7 @@ /*! * NO_FORWARD_PROGRESS_MAX : - * maximum allowed nb of calls to ZSTD_decompressStream() and ZSTD_decompress_generic() + * maximum allowed nb of calls to ZSTD_decompressStream() * without any forward progress * (defined as: no byte read from input, and no byte flushed to output) * before triggering an error. @@ -1565,13 +1565,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } - -size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - return ZSTD_decompressStream(dctx, output, input); -} - -size_t ZSTD_decompress_generic_simpleArgs ( +size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) @@ -1579,12 +1573,13 @@ size_t ZSTD_decompress_generic_simpleArgs ( ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; ZSTD_inBuffer input = { src, srcSize, *srcPos }; /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompress_generic(dctx, &output, &input); + size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); *dstPos = output.pos; *srcPos = input.pos; return cErr; } + size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) { if ( (reset == ZSTD_reset_session_only) diff --git a/lib/zstd.h b/lib/zstd.h index d5bd738f..0cc431c1 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -473,16 +473,15 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ***************************************/ /* API design : - * In this API, parameters are pushed one by one into an existing context, + * Parameters are pushed one by one into an existing context, * using ZSTD_CCtx_set*() functions. * Pushed parameters are sticky : they are applied to next job, and any subsequent job. - * Note that "sticky" parameters are only applicable with `ZSTD_compress2()` and `ZSTD_compressStream2()` ! - * They do not apply should the context be used with a "simple" variant such as ZSTD_compressCCtx() + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() * * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * - * This API gives access to all advanced capabilities. - * It supercedes all other "advanced" API entry points in the experimental section. + * This API supercedes all other "advanced" API entry points in the experimental section. * In the future, we expect to remove from experimental API entry points which are redundant with this API. */ @@ -621,7 +620,7 @@ typedef enum { * ZSTD_p_forceMaxWindow * ZSTD_p_forceAttachDict * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never use experimentalParam names directly + * note : never ever use experimentalParam? names directly */ ZSTD_p_experimentalParam1=500, ZSTD_p_experimentalParam2=10, @@ -799,12 +798,12 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, /* Advanced decompression API */ /* ============================== */ -/* The following API works the same way as the advanced compression API : - * a context is created, parameters are pushed into it one by one, - * then the context can be used to decompress data - * using ZSTD_decompress_generic(), similar to the streaming API. - * Note that sticky parameters only apply to ZSTD_decompress_generic() and ZSTD_decompress_generic_simpleArgs(). - * They are not valid if a "simple" function is used on the context (like `ZSTD_decompressDCtx()`). +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are "sticky, and remain valid for all futures decompression jobs + * started with the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : No new decompression function is provided, + * use existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). */ @@ -821,7 +820,7 @@ typedef enum { * At the time of this writing, they include : * ZSTD_p_format * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never use experimentalParam names directly + * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000 @@ -870,7 +869,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Note 1 : Currently, only one dictionary can be managed. * Referencing a new dictionary effectively "discards" any previous one. - * Special : adding a NULL DDict means "return to no-dictionary mode". + * Special: referencing a NULL DDict means "return to no-dictionary mode". * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. */ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); @@ -880,16 +879,16 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); * This is the reverse operation of ZSTD_CCtx_refPrefix(), * and must use the same prefix as the one used during compression. * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0. + * End of frame is reached when ZSTD_decompressStream() returns 0. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary * Note 2 : Prefix buffer is referenced. It **must** outlive decompression job. * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_DCtx_decompress_generic() returns 0. + * reached when ZSTD_decompressStream() returns 0. * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A fulldict prefix requires building tables, hence is more costly. + * A full dictionary is more costly, as it requires building tables. */ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); @@ -903,17 +902,6 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); -/*! ZSTD_decompress_generic() : - * Behave the same as ZSTD_decompressStream. - * Decompression parameters cannot be changed once decompression is started. - * @return : an error code, which can be tested using ZSTD_isError() - * if >0, a hint, nb of expected input bytes for next invocation. - * `0` means : a frame has just been fully decoded and flushed. - */ -ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input); - /**************************************************************************************** * experimental API (static linking only) @@ -1459,13 +1447,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); -/*! ZSTD_decompress_generic_simpleArgs() : - * Same as ZSTD_decompress_generic(), +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), * but using only integral types as arguments. * This can be helpful for binders from dynamic languages * which have troubles handling structures containing memory pointers. */ -ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs ( +ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos); diff --git a/programs/benchzstd.c b/programs/benchzstd.c index c0029657..75fecf8f 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -243,7 +243,7 @@ static size_t local_defaultDecompress( if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; } - moreToFlush = ZSTD_decompress_generic(dctx, &out, &in); + moreToFlush = ZSTD_decompressStream(dctx, &out, &in); if (ZSTD_isError(moreToFlush)) { return moreToFlush; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index dd6e260c..fc323cff 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1331,7 +1331,7 @@ static int basicUnitTests(U32 seed, double compressibility) } { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; - size_t const result = ZSTD_decompress_generic(dctx, &out, &in); + size_t const result = ZSTD_decompressStream(dctx, &out, &in); if (result != 0) goto _output_error; if (in.pos != in.size) goto _output_error; if (out.pos != inputSize) goto _output_error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a9e349df..a4bb906d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -948,7 +948,7 @@ static size_t local_defaultDecompress( if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; } - moreToFlush = ZSTD_decompress_generic(dctx, + moreToFlush = ZSTD_decompressStream(dctx, &out, &in); if (ZSTD_isError(moreToFlush)) { return moreToFlush; diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 38e6b769..19cf7cd7 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -691,7 +691,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (inBuff.pos != inBuff.size) goto _output_error; { ZSTD_outBuffer decOut = {decodedBuffer, size, 0}; ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0}; - CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) ); + CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) ); if (decIn.pos != decIn.size) goto _output_error; if (decOut.pos != size) goto _output_error; { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0); @@ -759,7 +759,7 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.src = compressedBuffer; inBuff.size = cSize; inBuff.pos = 0; - CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) ); + CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */ DISPLAYLEVEL(3, "OK \n");