added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize()
it complements ZSTD_estimateCCtxSize() for the special case of ZSTD_initCStream_usingDict()dev
parent
7855366598
commit
a1d6704d7f
|
@ -372,15 +372,17 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
|
|
||||||
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
|
|
||||||
</b><p> Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
|
|
||||||
`frameContentSize` is an optional parameter, provide `0` if unknown
|
|
||||||
</p></pre><BR>
|
|
||||||
|
|
||||||
<pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
<pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||||
</b><p> Create a ZSTD compression context using external alloc and free functions
|
</b><p> Create a ZSTD compression context using external alloc and free functions
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
|
||||||
|
</b><p> Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
|
||||||
|
Set streaming to 1 if the CCtx will be used for streaming (CStream).
|
||||||
|
Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
|
||||||
|
Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
||||||
</b><p> amount of used memory is variable, depending primarily on compression level
|
</b><p> amount of used memory is variable, depending primarily on compression level
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -406,6 +408,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
|
||||||
|
</b><p> Estimate amount of memory that will be needed to create a dictionary with following arguments
|
||||||
|
Note : if dictionary is created "byReference", reduce this amount by dictSize
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
<pre><b>size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
||||||
</b><p> Gives the amount of memory used by a given ZSTD_sizeof_CDict
|
</b><p> Gives the amount of memory used by a given ZSTD_sizeof_CDict
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -476,6 +483,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference
|
</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
|
||||||
|
<pre><b>size_t ZSTD_estimateDDictSize(size_t dictSize);
|
||||||
|
</b><p> Estimate amount of memory that will be needed to create a dictionary for decompression.
|
||||||
|
Note : if dictionary is created "byReference", reduce this amount by dictSize
|
||||||
|
</p></pre><BR>
|
||||||
|
|
||||||
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
<pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||||
</b><p> Gives the amount of memory used by a given ZSTD_DDict
|
</b><p> Gives the amount of memory used by a given ZSTD_DDict
|
||||||
</p></pre><BR>
|
</p></pre><BR>
|
||||||
|
@ -509,7 +521,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
|
||||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||||
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< same as ZSTD_sizeof_CCtx */<b>
|
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< same as ZSTD_sizeof_CCtx */<b>
|
||||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b>
|
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b>
|
||||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||||
|
|
|
@ -2984,6 +2984,14 @@ struct ZSTD_CDict_s {
|
||||||
ZSTD_CCtx* refContext;
|
ZSTD_CCtx* refContext;
|
||||||
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
|
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
|
||||||
|
|
||||||
|
/*! ZSTD_estimateCDictSize() :
|
||||||
|
* Estimate amount of memory that will be needed to create a dictionary with following arguments */
|
||||||
|
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
|
||||||
|
{
|
||||||
|
cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
|
||||||
|
return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0);
|
||||||
|
}
|
||||||
|
|
||||||
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
|
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
|
||||||
{
|
{
|
||||||
if (cdict==NULL) return 0; /* support sizeof on NULL */
|
if (cdict==NULL) return 0; /* support sizeof on NULL */
|
||||||
|
|
|
@ -1969,6 +1969,14 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*! ZSTD_estimateDDictSize() :
|
||||||
|
* Estimate amount of memory that will be needed to create a dictionary for decompression.
|
||||||
|
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
|
||||||
|
size_t ZSTD_estimateDDictSize(size_t dictSize)
|
||||||
|
{
|
||||||
|
return dictSize + sizeof(ZSTD_DDict);
|
||||||
|
}
|
||||||
|
|
||||||
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
||||||
{
|
{
|
||||||
if (ddict==NULL) return 0; /* support sizeof on NULL */
|
if (ddict==NULL) return 0; /* support sizeof on NULL */
|
||||||
|
|
24
lib/zstd.h
24
lib/zstd.h
|
@ -458,17 +458,17 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
|
||||||
/***************************************
|
/***************************************
|
||||||
* Advanced compression functions
|
* Advanced compression functions
|
||||||
***************************************/
|
***************************************/
|
||||||
/*! ZSTD_estimateCCtxSize() :
|
|
||||||
* Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
|
|
||||||
* Set streaming to 1 if the CCtx will be used for streaming (CStream)
|
|
||||||
* Note : this function is currently unable to estimate additional memory allocation needed to create an internal CDict
|
|
||||||
* which can only happen when starting with ZSTD_initCStream_usingDict() */
|
|
||||||
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
|
|
||||||
|
|
||||||
/*! ZSTD_createCCtx_advanced() :
|
/*! ZSTD_createCCtx_advanced() :
|
||||||
* Create a ZSTD compression context using external alloc and free functions */
|
* Create a ZSTD compression context using external alloc and free functions */
|
||||||
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
/*! ZSTD_estimateCCtxSize() :
|
||||||
|
* Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
|
||||||
|
* Set streaming to 1 if the CCtx will be used for streaming (CStream).
|
||||||
|
* Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
|
||||||
|
* Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */
|
||||||
|
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
|
||||||
|
|
||||||
/*! ZSTD_sizeofCCtx() :
|
/*! ZSTD_sizeofCCtx() :
|
||||||
* amount of used memory is variable, depending primarily on compression level */
|
* amount of used memory is variable, depending primarily on compression level */
|
||||||
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
||||||
|
@ -493,6 +493,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
|
||||||
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
|
||||||
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
|
ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
/*! ZSTD_estimateCDictSize() :
|
||||||
|
* Estimate amount of memory that will be needed to create a dictionary with following arguments
|
||||||
|
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
|
||||||
|
ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
|
||||||
|
|
||||||
/*! ZSTD_sizeof_CDict() :
|
/*! ZSTD_sizeof_CDict() :
|
||||||
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
|
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
|
||||||
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
|
||||||
|
@ -564,6 +569,11 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
|
||||||
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
||||||
unsigned byReference, ZSTD_customMem customMem);
|
unsigned byReference, ZSTD_customMem customMem);
|
||||||
|
|
||||||
|
/*! ZSTD_estimateDDictSize() :
|
||||||
|
* Estimate amount of memory that will be needed to create a dictionary for decompression.
|
||||||
|
* Note : if dictionary is created "byReference", reduce this amount by dictSize */
|
||||||
|
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
|
||||||
|
|
||||||
/*! ZSTD_sizeof_DDict() :
|
/*! ZSTD_sizeof_DDict() :
|
||||||
* Gives the amount of memory used by a given ZSTD_DDict */
|
* Gives the amount of memory used by a given ZSTD_DDict */
|
||||||
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||||
|
|
|
@ -404,6 +404,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||||
if (r != CNBuffSize) goto _output_error);
|
if (r != CNBuffSize) goto _output_error);
|
||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
|
||||||
|
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||||
|
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
|
||||||
|
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
|
||||||
|
}
|
||||||
|
|
||||||
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
|
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
|
||||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
|
||||||
ZSTD_customMem customMem = { NULL, NULL, NULL };
|
ZSTD_customMem customMem = { NULL, NULL, NULL };
|
||||||
|
|
Loading…
Reference in New Issue