refactored ZSTD_estimateDStreamSize()
now uses windowSize as argument. Also : created ZSTD_estimateDStreamSize_fromFrame()dev
parent
09ae03a570
commit
dde10b23fe
|
@ -332,7 +332,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||
</b></pre><BR>
|
||||
<pre><b>typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
size_t windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameHeader;
|
||||
|
@ -421,12 +421,17 @@ size_t ZSTD_estimateDCtxSize(void);
|
|||
|
||||
<pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel);
|
||||
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
|
||||
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
|
||||
size_t ZSTD_estimateDStreamSize(size_t windowSize);
|
||||
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
||||
</b><p> ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
|
||||
If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
|
||||
ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
Note : CStream estimation is only correct for single-threaded compression.
|
||||
ZSTD_DStream memory budget depends on window Size.
|
||||
This information can be passed manually, using ZSTD_estimateDStreamSize,
|
||||
or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
||||
Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
||||
an internal ?Dict will be created, which size is not estimated here.
|
||||
an internal ?Dict will be created, which additional size is not estimated here.
|
||||
In this case, get total size by adding ZSTD_estimate?DictSize
|
||||
</p></pre><BR>
|
||||
|
||||
|
@ -622,6 +627,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
|
|||
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
|
||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b>
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */<b>
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
|
@ -640,6 +646,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
|
|||
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticDCtx() */<b>
|
||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
|
||||
|
@ -694,16 +701,18 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
|
|||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
|
||||
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID used.
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID in use.
|
||||
(Note : content size is optional, it may not be present. 0 means : content size unknown).
|
||||
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
|
||||
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
|
||||
Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
|
||||
Frame parameters are extracted from the beginning of the compressed frame.
|
||||
Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
|
||||
Each application can set its own limit, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support windowSize of at least 8 MB.
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
|
@ -747,7 +756,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
|
|||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
<BR></pre>
|
||||
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input, see details below */<b>
|
||||
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
|
||||
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
|
|
@ -3594,6 +3594,11 @@ ZSTD_CStream* ZSTD_createCStream(void)
|
|||
return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
|
||||
{
|
||||
return ZSTD_initStaticCCtx(workspace, workspaceSize);
|
||||
}
|
||||
|
||||
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
|
||||
{ /* CStream and CCtx are now same object */
|
||||
return ZSTD_createCCtx_advanced(customMem);
|
||||
|
|
|
@ -2143,6 +2143,11 @@ ZSTD_DStream* ZSTD_createDStream(void)
|
|||
return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
|
||||
}
|
||||
|
||||
ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
|
||||
{
|
||||
return ZSTD_initStaticDCtx(workspace, workspaceSize);
|
||||
}
|
||||
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
return ZSTD_createDCtx_advanced(customMem);
|
||||
|
@ -2214,15 +2219,23 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
|
|||
return ZSTD_sizeof_DCtx(zds);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
|
||||
size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
||||
{
|
||||
size_t const windowSize = fHeader.windowSize;
|
||||
size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
size_t const inBuffSize = blockSize; /* no block can be larger */
|
||||
size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
|
||||
return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
||||
}
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_frameHeader fh;
|
||||
size_t const err = ZSTD_getFrameHeader(&fh, src, srcSize);
|
||||
if (ZSTD_isError(err)) return err;
|
||||
if (err>0) return ERROR(srcSize_wrong);
|
||||
return ZSTD_estimateDStreamSize(fh.windowSize);
|
||||
}
|
||||
|
||||
|
||||
/* ***** Decompression ***** */
|
||||
|
||||
|
|
31
lib/zstd.h
31
lib/zstd.h
|
@ -412,7 +412,7 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
size_t windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTD_frameHeader;
|
||||
|
@ -509,12 +509,17 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
|
|||
* ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
|
||||
* If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
|
||||
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
|
||||
* Note : CStream estimation is only correct for single-threaded compression.
|
||||
* ZSTD_DStream memory budget depends on window Size.
|
||||
* This information can be passed manually, using ZSTD_estimateDStreamSize,
|
||||
* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
|
||||
* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
|
||||
* an internal ?Dict will be created, which size is not estimated here.
|
||||
* an internal ?Dict will be created, which additional size is not estimated here.
|
||||
* In this case, get total size by adding ZSTD_estimate?DictSize */
|
||||
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
|
||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_estimate?DictSize() :
|
||||
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
|
||||
|
@ -717,6 +722,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
|
@ -737,6 +743,7 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
|
|||
/*===== Advanced Streaming decompression functions =====*/
|
||||
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
|
||||
|
@ -803,16 +810,18 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
|||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
|
||||
It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
|
||||
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID used.
|
||||
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
|
||||
such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID in use.
|
||||
(Note : content size is optional, it may not be present. 0 means : content size unknown).
|
||||
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
|
||||
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
|
||||
Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
|
||||
Frame parameters are extracted from the beginning of the compressed frame.
|
||||
Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
|
||||
Each application can set its own limit, depending on local restrictions.
|
||||
For extended interoperability, it is recommended to support windowSize of at least 8 MB.
|
||||
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
|
||||
Data fragment must be large enough to ensure successful decoding.
|
||||
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
|
||||
@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTD_isError().
|
||||
|
||||
|
@ -857,7 +866,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
|
|||
*/
|
||||
|
||||
/*===== Buffer-less streaming decompression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
|
||||
ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
|
|
@ -285,8 +285,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||
const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
|
||||
size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
|
||||
if (gfhError!=0) goto _output_error;
|
||||
DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
|
||||
{ size_t const s = ZSTD_estimateDStreamSize(fhi)
|
||||
DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize);
|
||||
{ size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
|
||||
/* uses ZSTD_initDStream_usingDict() */
|
||||
+ ZSTD_estimateDDictSize(dictSize, 0);
|
||||
if (ZSTD_isError(s)) goto _output_error;
|
||||
|
|
Loading…
Reference in New Issue