facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated erroneous comments using ZSTD_dm_* 

instead of the current ZSTD_dct_*,
reported by @nigeltao (#1822)
dev
Yann Collet 2019-10-16 16:14:04 -07:00
parent 8b6d96827c
commit 6323966e53
4 changed files with 40 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
doc/zstd_manual.html
View File

@ -796,7 +796,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU consuming operation, with non-negligible impact on latency. It's a CPU consuming operation, with non-negligible impact on latency.
If there is a need to use the same prefix multiple times, consider loadDictionary instead. If there is a need to use the same prefix multiple times, consider loadDictionary instead.
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.
</p></pre><BR> </p></pre><BR>
@ -840,7 +840,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note 2 : Prefix buffer is referenced. It **must** outlive decompression. Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
Prefix buffer must remain unmodified up to the end of frame, Prefix buffer must remain unmodified up to the end of frame,
reached when ZSTD_decompressStream() returns 0. reached when ZSTD_decompressStream() returns 0.
Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
A full dictionary is more costly, as it requires building tables. A full dictionary is more costly, as it requires building tables.
@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
<BR></pre> <BR></pre>
<pre><b>typedef struct {
unsigned int matchPos; </b>/* Match pos in dst */<b>
</b>/* If seqDef.offset > 3, then this is seqDef.offset - 3<b>
* If seqDef.offset < 3, then this is the corresponding repeat offset
* But if seqDef.offset < 3 and litLength == 0, this is the
* repeat offset before the corresponding repeat offset
* And if seqDef.offset == 3 and litLength == 0, this is the
* most recent repeat offset - 1
*/
unsigned int offset;
unsigned int litLength; </b>/* Literal length */<b>
unsigned int matchLength; </b>/* Match length */<b>
</b>/* 0 when seq not rep and seqDef.offset otherwise<b>
* when litLength == 0 this will be <= 4, otherwise <= 3 like normal
*/
unsigned int rep;
} ZSTD_Sequence;
</b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b> unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b> unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small) or an error code (if srcSize is too small)
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize);
</b><p> Extract sequences from the sequence store
zc can be used to insert custom compression params.
This function invokes ZSTD_compress2
@return : number of sequences extracted
</p></pre><BR>
<a name="Chapter16"></a><h2>Memory management</h2><pre></pre> <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
@ -1322,7 +1349,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne
* *
* Creates of an internal CDict (incompatible with static CCtx), except if * Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used. * dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
*/ */
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@ -1337,7 +1364,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
* *
* pledgedSrcSize must be correct. If srcSize is not known at init time, use * pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
*/ */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize);

2
lib/compress/zstd_compress.c
View File

@ -3518,7 +3518,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
/* ZSTD_initCStream_advanced() : /* ZSTD_initCStream_advanced() :
* pledgedSrcSize must be exact. * pledgedSrcSize must be exact.
* if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pss) ZSTD_parameters params, unsigned long long pss)

8
lib/zstd.h
View File

@ -928,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU consuming operation, with non-negligible impact on latency. * It's a CPU consuming operation, with non-negligible impact on latency.
* If there is a need to use the same prefix multiple times, consider loadDictionary instead. * If there is a need to use the same prefix multiple times, consider loadDictionary instead.
* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize); const void* prefix, size_t prefixSize);
@ -972,7 +972,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
* Note 2 : Prefix buffer is referenced. It **must** outlive decompression. * Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
* Prefix buffer must remain unmodified up to the end of frame, * Prefix buffer must remain unmodified up to the end of frame,
* reached when ZSTD_decompressStream() returns 0. * reached when ZSTD_decompressStream() returns 0.
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
* A full dictionary is more costly, as it requires building tables. * A full dictionary is more costly, as it requires building tables.
@ -1670,7 +1670,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
* *
* Creates of an internal CDict (incompatible with static CCtx), except if * Creates of an internal CDict (incompatible with static CCtx), except if
* dict == NULL or dictSize < 8, in which case no dict is used. * dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
*/ */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@ -1685,7 +1685,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
* *
* pledgedSrcSize must be correct. If srcSize is not known at init time, use * pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
*/ */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize);

8
tests/fuzzer.c
View File

@ -1387,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error; if (cdict==NULL) goto _output_error;
@ -1395,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error; if (cdict!=NULL) goto _output_error;
@ -1403,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++); DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
{ {
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@ -1417,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++); DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
{ {
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);