facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #891 from facebook/contentSize

Browse Source 
Content size
This commit is contained in:
Yann Collet 2017-10-17 17:24:51 -07:00 committed by GitHub
commit 1ff8a8c109
10 changed files with 149 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/zstd_manual.html
View File

@ -631,10 +631,10 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b> ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b>
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b> size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b> ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b> size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b> size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
</pre></b><BR> </pre></b><BR>
@ -642,9 +642,11 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
</b><p> start a new compression job, using same parameters from previous job. </b><p> start a new compression job, using same parameters from previous job.
This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
Note that zcs must be init at least once before using ZSTD_resetCStream(). Note that zcs must be init at least once before using ZSTD_resetCStream().
pledgedSrcSize==0 means "srcSize unknown". If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
@return : 0, or an error code (which can be tested using ZSTD_isError()) For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
@return : 0, or an error code (which can be tested using ZSTD_isError())
</p></pre><BR> </p></pre><BR>
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
@ -693,10 +695,10 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression para
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b> size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b> size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b> size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> <a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
@ -848,11 +850,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
* Special: value 0 means "do not change strategy". */ * Special: value 0 means "do not change strategy". */
</b>/* frame parameters */<b> </b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1)<b> ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
* note that content size must be known at the beginning, * Content size must be known at the beginning of compression,
* it is sent using ZSTD_CCtx_setPledgedSrcSize() */ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b> ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b> ZSTD_p_dictIDFlag, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
</b>/* multi-threading parameters */<b> </b>/* multi-threading parameters */<b>
ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b> ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b>
@ -913,7 +915,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
@result : 0, or an error code (which can be tested with ZSTD_isError()). @result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : 0 means zero, empty. Note 1 : 0 means zero, empty.
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs. ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
Note 2 : If all data is provided and consumed in a single round, Note 2 : If all data is provided and consumed in a single round,
this value is overriden by srcSize instead. this value is overriden by srcSize instead.
</p></pre><BR> </p></pre><BR>

56
lib/compress/zstd_compress.c
View File

@ -78,6 +78,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
if (!cctx) return NULL; if (!cctx) return NULL;
cctx->customMem = customMem; cctx->customMem = customMem;
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
cctx->requestedParams.fParams.contentSizeFlag = 1;
ZSTD_STATIC_ASSERT(zcss_init==0); ZSTD_STATIC_ASSERT(zcss_init==0);
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
return cctx; return cctx;
@ -391,7 +392,7 @@ size_t ZSTD_CCtxParam_setParameter(
case ZSTD_p_contentSizeFlag : case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */ /* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(5, "set content size flag = %u", (value>0)); DEBUGLOG(4, "set content size flag = %u", (value>0));
params->fParams.contentSizeFlag = value > 0; params->fParams.contentSizeFlag = value > 0;
return 0; return 0;
@ -401,7 +402,7 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; return 0;
case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0); params->fParams.noDictIDFlag = (value == 0);
return 0; return 0;
@ -530,7 +531,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
cctx->cdict = NULL; cctx->cdict = NULL;
} else { } else {
ZSTD_compressionParameters const cParams = ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize); ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced( cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize, dict, dictSize,
dictLoadMethod, dictMode, dictLoadMethod, dictMode,
@ -821,9 +822,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_compResetPolicy_e const crp, ZSTD_compResetPolicy_e const crp,
ZSTD_buffered_policy_e const zbuff) ZSTD_buffered_policy_e const zbuff)
{ {
DEBUGLOG(4, "ZSTD_resetCCtx_internal"); DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u", (U32)pledgedSrcSize);
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize);
if (crp == ZSTDcrp_continue) { if (crp == ZSTDcrp_continue) {
if (ZSTD_equivalentParams(params, zc->appliedParams)) { if (ZSTD_equivalentParams(params, zc->appliedParams)) {
@ -1002,7 +1002,7 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
* The "context", in this case, refers to the hash and chain tables, entropy * The "context", in this case, refers to the hash and chain tables, entropy
* tables, and dictionary offsets. * tables, and dictionary offsets.
* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
* pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1 * pledgedSrcSize=0 means "empty".
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
const ZSTD_CCtx* srcCCtx, const ZSTD_CCtx* srcCCtx,
@ -1059,7 +1059,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0); ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
fParams.contentSizeFlag = pledgedSrcSize>0; if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff); return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff);
} }
@ -2070,6 +2071,8 @@ size_t ZSTD_compressBegin_advanced_internal(
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
@ -2096,7 +2099,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
cctxParams, 0, ZSTDb_not_buffered); cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
} }
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
@ -2298,7 +2301,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem) ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{ {
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@ -2373,9 +2376,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
+ cctxSize; + cctxSize;
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
void* ptr; void* ptr;
DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
if ((size_t)workspace & 7) return NULL; /* 8-aligned */ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
(U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
if (workspaceSize < neededSize) return NULL; if (workspaceSize < neededSize) return NULL;
@ -2411,7 +2414,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
{ ZSTD_CCtx_params params = cctx->requestedParams; { ZSTD_CCtx_params params = cctx->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict); params.cParams = ZSTD_getCParamsFromCDict(cdict);
params.fParams = fParams; params.fParams = fParams;
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
return ZSTD_compressBegin_internal(cctx, return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto, NULL, 0, ZSTD_dm_auto,
cdict, cdict,
@ -2426,7 +2429,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{ {
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
} }
@ -2515,12 +2518,15 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
return 0; /* ready to go */ return 0; /* ready to go */
} }
/* ZSTD_resetCStream():
* pledgedSrcSize == 0 means "unknown" */
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{ {
ZSTD_CCtx_params params = zcs->requestedParams; ZSTD_CCtx_params params = zcs->requestedParams;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0); DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize);
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
params.fParams.contentSizeFlag = 1;
params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
DEBUGLOG(4, "ZSTD_resetCStream");
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
} }
@ -2587,13 +2593,20 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */ return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */
} }
/* ZSTD_initCStream_advanced() :
* pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize) ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
(U32)pledgedSrcSize, params.fParams.contentSizeFlag);
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0))
pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
} }
@ -2602,7 +2615,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
} }
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
@ -2816,18 +2829,18 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
} }
DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads); DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads);
CHECK_F( ZSTDMT_initCStream_internal( CHECK_F( ZSTDMT_initCStream_internal(
cctx->mtctx, cctx->mtctx,
prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
cctx->streamStage = zcss_load; cctx->streamStage = zcss_load;
cctx->appliedParams.nbThreads = params.nbThreads; cctx->appliedParams.nbThreads = params.nbThreads;
} else } else
#endif #endif
{ { CHECK_F( ZSTD_resetCStream_internal(
CHECK_F( ZSTD_resetCStream_internal(
cctx, prefixDict.dict, prefixDict.dictSize, cctx, prefixDict.dict, prefixDict.dictSize,
prefixDict.dictMode, cctx->cdict, params, prefixDict.dictMode, cctx->cdict, params,
cctx->pledgedSrcSizePlusOne-1) ); cctx->pledgedSrcSizePlusOne-1) );
assert(cctx->streamStage == zcss_load);
} } } }
/* compression stage */ /* compression stage */
@ -3053,5 +3066,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));
params.cParams = cParams; params.cParams = cParams;
params.fParams.contentSizeFlag = 1;
return params; return params;
} }

41
lib/compress/zstdmt_compress.c
View File

@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool); ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
const void* const src = (const char*)job->srcStart + job->dictSize; const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t dstBuff = job->dstBuff; buffer_t dstBuff = job->dstBuff;
DEBUGLOG(5, "ZSTDMT_compressChunk")
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription)
job->dstBuff = dstBuff; job->dstBuff = dstBuff;
} }
if (job->cdict) { /* should only happen for first segment */ if (job->cdict) {
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize); size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
DEBUGLOG(5, "using CDict"); DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict");
assert(job->firstChunk); /* should only happen for first segment */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */ } else { /* srcStart points at reloaded section */
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ ZSTD_CCtx_params jobParams = job->params;
{ ZSTD_CCtx_params jobParams = job->params; size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
size_t const forceWindowError = U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); /* load dictionary in "content-only" mode (no header analysis) */
/* Force loading dictionary in "content-only" mode (no header analysis) */ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize); if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { job->cSize = initError;
job->cSize = initError; goto _endJob;
goto _endJob; }
} }
} }
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0); size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
} }
DEBUGLOG(5, "Compressing : "); DEBUGLOG(5, "Compressing : ");
DEBUG_PRINTHEX(4, job->srcStart, 12); DEBUG_PRINTHEX(6, job->srcStart, 12);
job->cSize = (job->lastChunk) ? job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize); ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal(
assert(jobParams.nbThreads == 0); assert(jobParams.nbThreads == 0);
assert(mtctx->cctxPool->totalCCtx == params.nbThreads); assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
if (nbChunks==1) { /* fallback to single-thread mode */ if (nbChunks==1) { /* fallback to single-thread mode */
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal(
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
mtctx->jobs[u].dictSize = dictSize; mtctx->jobs[u].dictSize = dictSize;
mtctx->jobs[u].srcSize = chunkSize; mtctx->jobs[u].srcSize = chunkSize;
mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL; mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
mtctx->jobs[u].fullFrameSize = srcSize; mtctx->jobs[u].fullFrameSize = srcSize;
mtctx->jobs[u].params = jobParams; mtctx->jobs[u].params = jobParams;
/* do not calculate checksum within sections, but write it in header for first section */ /* do not calculate checksum within sections, but write it in header for first section */
@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->params = params; zcs->params = params;
zcs->frameContentSize = pledgedSrcSize; zcs->frameContentSize = pledgedSrcSize;
if (dict) { if (dict) {
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
} else { } else {
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = NULL; zcs->cdictLocal = NULL;
zcs->cdict = cdict; zcs->cdict = cdict;
@ -838,9 +838,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
/* ZSTDMT_resetCStream() : /* ZSTDMT_resetCStream() :
* pledgedSrcSize is optional and can be zero == unknown */ * pledgedSrcSize can be zero == unknown (for the time being)
* prefer using ZSTD_CONTENTSIZE_UNKNOWN,
* as `0` might mean "empty" in the future */
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize) size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
{ {
if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
if (zcs->params.nbThreads==1) if (zcs->params.nbThreads==1)
return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize); return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params, return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params,
@ -852,7 +855,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
ZSTD_CCtx_params cctxParams = zcs->params; ZSTD_CCtx_params cctxParams = zcs->params;
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0); return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
} }

2
lib/compress/zstdmt_compress.h
View File

@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
/* === Streaming functions === */ /* === Streaming functions === */
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);

26
lib/zstd.h
View File

@ -731,10 +731,10 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/ /*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
@ -742,9 +742,11 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
* start a new compression job, using same parameters from previous job. * start a new compression job, using same parameters from previous job.
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
* Note that zcs must be init at least once before using ZSTD_resetCStream(). * Note that zcs must be init at least once before using ZSTD_resetCStream().
* pledgedSrcSize==0 means "srcSize unknown". * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */ * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
* but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
@ -800,10 +802,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress
/*===== Buffer-less streaming compression functions =====*/ /*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@ -1000,11 +1002,11 @@ typedef enum {
* Special: value 0 means "do not change strategy". */ * Special: value 0 means "do not change strategy". */
/* frame parameters */ /* frame parameters */
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
* note that content size must be known at the beginning, * Content size must be known at the beginning of compression,
* it is sent using ZSTD_CCtx_setPledgedSrcSize() */ * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */ ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
/* multi-threading parameters */ /* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1) ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@ -1066,7 +1068,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Note 1 : 0 means zero, empty. * Note 1 : 0 means zero, empty.
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
* Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs. * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
* Note 2 : If all data is provided and consumed in a single round, * Note 2 : If all data is provided and consumed in a single round,
* this value is overriden by srcSize instead. */ * this value is overriden by srcSize instead. */
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);

4
programs/Makefile
View File

@ -37,8 +37,8 @@ endif
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder \ -I$(ZSTDDIR)/dictBuilder \
-DZSTD_NEWAPI \ -DXXH_NAMESPACE=ZSTD_ \
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library -DZSTD_NEWAPI
CFLAGS ?= -O3 CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \

28
programs/fileio.c
View File

@ -439,9 +439,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
#ifdef ZSTD_NEWAPI #ifdef ZSTD_NEWAPI
{ /* frame parameters */ { /* frame parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
(void)srcSize;
/* compression level */ /* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
/* long distance matching */ /* long distance matching */
@ -467,7 +467,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
} }
#elif defined(ZSTD_MULTITHREAD) #elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
@ -815,11 +817,11 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */ /* init */
#ifdef ZSTD_NEWAPI #ifdef ZSTD_NEWAPI
if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */ if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); /* note : fileSize==0 means "empty" */ ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
#elif defined(ZSTD_MULTITHREAD) #elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
#else #else
CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */ CHECK( ZSTD_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
#endif #endif
/* Main compression loop */ /* Main compression loop */
@ -1005,7 +1007,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
size_t dfnSize = FNSPACE; size_t dfnSize = FNSPACE;
char* dstFileName = (char*)malloc(FNSPACE); char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0; size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : UTIL_getFileSize(inFileNamesTable[0]) ;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
/* init */ /* init */
@ -1031,9 +1033,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
free(dstFileName); free(dstFileName);
dfnSize = ifnSize + 20; dfnSize = ifnSize + 20;
dstFileName = (char*)malloc(dfnSize); dstFileName = (char*)malloc(dfnSize);
if (!dstFileName) if (!dstFileName) {
EXM_THROW(30, "zstd: %s", strerror(errno)); EXM_THROW(30, "zstd: %s", strerror(errno));
} } }
strcpy(dstFileName, inFileNamesTable[u]); strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix); strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel); missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
@ -1926,7 +1928,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
return detectError; return detectError;
} }
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){ static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
double const windowSizeUnit = (double)info->windowSize / unit; double const windowSizeUnit = (double)info->windowSize / unit;
@ -1949,8 +1951,10 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
checkString, inFileName); checkString, inFileName);
} }
} else { } else {
DISPLAYOUT("%s \n", inFileName);
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); if (info->numSkippableFrames)
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n",
windowSizeUnit, unitStr, windowSizeUnit, unitStr,
(unsigned long long)info->windowSize); (unsigned long long)info->windowSize);
@ -1982,7 +1986,6 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
} }
static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){ static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
/* initialize info to avoid warnings */
fileInfo_t info; fileInfo_t info;
memset(&info, 0, sizeof(info)); memset(&info, 0, sizeof(info));
{ int const error = getFileInfo(&info, inFileName); { int const error = getFileInfo(&info, inFileName);
@ -2022,7 +2025,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
for (u=0; u<numFiles;u++) { for (u=0; u<numFiles;u++) {
error |= FIO_listFile(&total, filenameTable[u], displayLevel); error |= FIO_listFile(&total, filenameTable[u], displayLevel);
} }
if (numFiles > 1 && displayLevel <= 2) { if (numFiles > 1 && displayLevel <= 2) { /* display total */
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)total.compressedSize / unit; double const compressedSizeUnit = (double)total.compressedSize / unit;
@ -2042,8 +2045,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
total.numSkippableFrames, total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, total.nbFiles); ratio, checkString, total.nbFiles);
} } }
}
return error; return error;
} }
} }

2
tests/fullbench.c
View File

@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{ {
(void)buff2; (void)buff2;
ZSTD_compressBegin(g_zcc, 1); ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
} }

76
tests/playTests.sh
View File

@ -91,7 +91,7 @@ else
hasMT="true" hasMT="true"
fi fi
$ECHO "\n**** simple tests **** " $ECHO "\n===> simple tests "
./datagen > tmp ./datagen > tmp
$ECHO "test : basic compression " $ECHO "test : basic compression "
@ -166,7 +166,7 @@ $ZSTD -f tmp && die "tmp not present : should have failed"
test ! -f tmp.zst # tmp.zst should not be created test ! -f tmp.zst # tmp.zst should not be created
$ECHO "\n**** Advanced compression parameters **** " $ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
@ -180,14 +180,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
roundTripTest -g512K 19 roundTripTest -g512K 19
$ECHO "\n**** Pass-Through mode **** " $ECHO "\n===> Pass-Through mode "
$ECHO "Hello world 1!" | $ZSTD -df $ECHO "Hello world 1!" | $ZSTD -df
$ECHO "Hello world 2!" | $ZSTD -dcf $ECHO "Hello world 2!" | $ZSTD -dcf
$ECHO "Hello world 3!" > tmp1 $ECHO "Hello world 3!" > tmp1
$ZSTD -dcf tmp1 $ZSTD -dcf tmp1
$ECHO "\n**** frame concatenation **** " $ECHO "\n===> frame concatenation "
$ECHO "hello " > hello.tmp $ECHO "hello " > hello.tmp
$ECHO "world!" > world.tmp $ECHO "world!" > world.tmp
@ -218,7 +218,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
$ECHO "\n**** symbolic link test **** " $ECHO "\n===> symbolic link test "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp $ECHO "hello world" > hello.tmp
@ -233,7 +233,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi fi
$ECHO "\n**** test sparse file support **** " $ECHO "\n===> test sparse file support "
./datagen -g5M -P100 > tmpSparse ./datagen -g5M -P100 > tmpSparse
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
@ -260,7 +260,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse* rm tmpSparse*
$ECHO "\n**** multiple files tests **** " $ECHO "\n===> multiple files tests "
./datagen -s1 > tmp1 2> $INTOVOID ./datagen -s1 > tmp1 2> $INTOVOID
./datagen -s2 -g100K > tmp2 2> $INTOVOID ./datagen -s2 -g100K > tmp2 2> $INTOVOID
@ -283,7 +283,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
$ECHO "\n**** dictionary tests **** " $ECHO "\n===> dictionary tests "
$ECHO "- test with raw dict (content only) " $ECHO "- test with raw dict (content only) "
./datagen > tmpDict ./datagen > tmpDict
@ -339,7 +339,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
rm tmp* rm tmp*
$ECHO "\n**** cover dictionary tests **** " $ECHO "\n===> cover dictionary builder : advanced options "
TESTFILE=../programs/zstdcli.c TESTFILE=../programs/zstdcli.c
./datagen > tmpDict ./datagen > tmpDict
@ -359,7 +359,7 @@ $ECHO "- Create dictionary with size limit"
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp* rm tmp*
$ECHO "\n**** legacy dictionary tests **** " $ECHO "\n===> legacy dictionary builder "
TESTFILE=../programs/zstdcli.c TESTFILE=../programs/zstdcli.c
./datagen > tmpDict ./datagen > tmpDict
@ -380,7 +380,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp* rm tmp*
$ECHO "\n**** integrity tests **** " $ECHO "\n===> integrity tests "
$ECHO "test one file (tmp1.zst) " $ECHO "test one file (tmp1.zst) "
./datagen > tmp1 ./datagen > tmp1
@ -405,13 +405,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
$ECHO "\n**** golden files tests **** " $ECHO "\n===> golden files tests "
$ZSTD -t -r files $ZSTD -t -r files
$ZSTD -c -r files | $ZSTD -t $ZSTD -c -r files | $ZSTD -t
$ECHO "\n**** benchmark mode tests **** " $ECHO "\n===> benchmark mode tests "
$ECHO "bench one file" $ECHO "bench one file"
./datagen > tmp1 ./datagen > tmp1
@ -422,7 +422,7 @@ $ECHO "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1 $ZSTD -rqi1b1e2 tmp1
$ECHO "\n**** gzip compatibility tests **** " $ECHO "\n===> gzip compatibility tests "
GZIPMODE=1 GZIPMODE=1
$ZSTD --format=gzip -V || GZIPMODE=0 $ZSTD --format=gzip -V || GZIPMODE=0
@ -445,7 +445,7 @@ else
fi fi
$ECHO "\n**** gzip frame tests **** " $ECHO "\n===> gzip frame tests "
if [ $GZIPMODE -eq 1 ]; then if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp ./datagen > tmp
@ -459,7 +459,7 @@ else
fi fi
$ECHO "\n**** xz compatibility tests **** " $ECHO "\n===> xz compatibility tests "
LZMAMODE=1 LZMAMODE=1
$ZSTD --format=xz -V || LZMAMODE=0 $ZSTD --format=xz -V || LZMAMODE=0
@ -505,7 +505,7 @@ else
fi fi
$ECHO "\n**** xz frame tests **** " $ECHO "\n===> xz frame tests "
if [ $LZMAMODE -eq 1 ]; then if [ $LZMAMODE -eq 1 ]; then
./datagen > tmp ./datagen > tmp
@ -520,7 +520,7 @@ else
$ECHO "xz mode not supported" $ECHO "xz mode not supported"
fi fi
$ECHO "\n**** lz4 compatibility tests **** " $ECHO "\n===> lz4 compatibility tests "
LZ4MODE=1 LZ4MODE=1
$ZSTD --format=lz4 -V || LZ4MODE=0 $ZSTD --format=lz4 -V || LZ4MODE=0
@ -543,7 +543,7 @@ else
fi fi
$ECHO "\n**** lz4 frame tests **** " $ECHO "\n===> lz4 frame tests "
if [ $LZ4MODE -eq 1 ]; then if [ $LZ4MODE -eq 1 ]; then
./datagen > tmp ./datagen > tmp
@ -556,7 +556,7 @@ else
$ECHO "lz4 mode not supported" $ECHO "lz4 mode not supported"
fi fi
$ECHO "\n**** zstd round-trip tests **** " $ECHO "\n===> zstd round-trip tests "
roundTripTest roundTripTest
roundTripTest -g15K # TableID==3 roundTripTest -g15K # TableID==3
@ -569,7 +569,7 @@ roundTripTest -g516K 19 # btopt
fileRoundTripTest -g500K fileRoundTripTest -g500K
$ECHO "\n**** zstd long distance matching round-trip tests **** " $ECHO "\n===> zstd long distance matching round-trip tests "
roundTripTest -g0 "2 --long" roundTripTest -g0 "2 --long"
roundTripTest -g1000K "1 --long" roundTripTest -g1000K "1 --long"
roundTripTest -g517K "6 --long" roundTripTest -g517K "6 --long"
@ -580,45 +580,43 @@ fileRoundTripTest -g5M "3 --long"
if [ -n "$hasMT" ] if [ -n "$hasMT" ]
then then
$ECHO "\n**** zstdmt round-trip tests **** " $ECHO "\n===> zstdmt round-trip tests "
roundTripTest -g4M "1 -T0" roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2" roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2" roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M" fileRoundTripTest -g4M "19 -T2 -B1M"
$ECHO "\n**** zstdmt long distance matching round-trip tests **** " $ECHO "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long -T2" roundTripTest -g8M "3 --long -T2"
else else
$ECHO "\n**** no multithreading, skipping zstdmt tests **** " $ECHO "\n===> no multithreading, skipping zstdmt tests "
fi fi
rm tmp* rm tmp*
$ECHO "\n**** zstd --list/-l single frame tests ****" $ECHO "\n===> zstd --list/-l single frame tests "
./datagen > tmp1 ./datagen > tmp1
./datagen > tmp2 ./datagen > tmp2
./datagen > tmp3 ./datagen > tmp3
$ZSTD tmp* $ZSTD tmp*
$ZSTD -l *.zst $ZSTD -l *.zst
$ZSTD -lv *.zst $ZSTD -lv *.zst | grep "Decompressed Size:" # check that decompressed size is present in header
$ZSTD --list *.zst $ZSTD --list *.zst
$ZSTD --list -v *.zst $ZSTD --list -v *.zst
$ECHO "\n**** zstd --list/-l multiple frame tests ****" $ECHO "\n===> zstd --list/-l multiple frame tests "
cat tmp1.zst tmp2.zst > tmp12.zst cat tmp1.zst tmp2.zst > tmp12.zst
cat tmp12.zst tmp3.zst > tmp123.zst cat tmp12.zst tmp3.zst > tmp123.zst
$ZSTD -l *.zst $ZSTD -l *.zst
$ZSTD -lv *.zst $ZSTD -lv *.zst
$ZSTD --list *.zst
$ZSTD --list -v *.zst
$ECHO "\n**** zstd --list/-l error detection tests ****" $ECHO "\n===> zstd --list/-l error detection tests "
! $ZSTD -l tmp1 tmp1.zst ! $ZSTD -l tmp1 tmp1.zst
! $ZSTD --list tmp* ! $ZSTD --list tmp*
! $ZSTD -lv tmp1* ! $ZSTD -lv tmp1*
! $ZSTD --list -v tmp2 tmp12.zst ! $ZSTD --list -v tmp2 tmp12.zst
$ECHO "\n**** zstd --list/-l test with null files ****" $ECHO "\n===> zstd --list/-l test with null files "
./datagen -g0 > tmp5 ./datagen -g0 > tmp5
$ZSTD tmp5 $ZSTD tmp5
$ZSTD -l tmp5.zst $ZSTD -l tmp5.zst
@ -626,12 +624,12 @@ $ZSTD -l tmp5.zst
$ZSTD -lv tmp5.zst $ZSTD -lv tmp5.zst
! $ZSTD -lv tmp5* ! $ZSTD -lv tmp5*
$ECHO "\n**** zstd --list/-l test with no content size field ****" $ECHO "\n===> zstd --list/-l test with no content size field "
./datagen -g1MB | $ZSTD > tmp6.zst ./datagen -g513K | $ZSTD > tmp6.zst
$ZSTD -l tmp6.zst $ZSTD -l tmp6.zst
$ZSTD -lv tmp6.zst ! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header
$ECHO "\n**** zstd --list/-l test with no checksum ****" $ECHO "\n===> zstd --list/-l test with no checksum "
$ZSTD -f --no-check tmp1 $ZSTD -f --no-check tmp1
$ZSTD -l tmp1.zst $ZSTD -l tmp1.zst
$ZSTD -lv tmp1.zst $ZSTD -lv tmp1.zst
@ -639,7 +637,7 @@ $ZSTD -lv tmp1.zst
rm tmp* rm tmp*
$ECHO "\n**** zstd long distance matching tests **** " $ECHO "\n===> zstd long distance matching tests "
roundTripTest -g0 " --long" roundTripTest -g0 " --long"
roundTripTest -g9M "2 --long" roundTripTest -g9M "2 --long"
# Test parameter parsing # Test parameter parsing
@ -654,7 +652,7 @@ if [ "$1" != "--test-large-data" ]; then
exit 0 exit 0
fi fi
$ECHO "\n**** large files tests **** " $ECHO "\n===> large files tests "
roundTripTest -g270000000 1 roundTripTest -g270000000 1
roundTripTest -g250000000 2 roundTripTest -g250000000 2
@ -685,7 +683,7 @@ roundTripTest -g5000000000 -P99 1
fileRoundTripTest -g4193M -P99 1 fileRoundTripTest -g4193M -P99 1
$ECHO "\n**** zstd long, long distance matching round-trip tests **** " $ECHO "\n===> zstd long, long distance matching round-trip tests "
roundTripTest -g270000000 "1 --long" roundTripTest -g270000000 "1 --long"
roundTripTest -g130000000 -P60 "5 --long" roundTripTest -g130000000 -P60 "5 --long"
roundTripTest -g35000000 -P70 "8 --long" roundTripTest -g35000000 -P70 "8 --long"
@ -697,7 +695,7 @@ roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"
if [ -n "$hasMT" ] if [ -n "$hasMT" ]
then then
$ECHO "\n**** zstdmt long round-trip tests **** " $ECHO "\n===> zstdmt long round-trip tests "
roundTripTest -g80000000 -P99 "19 -T2" " " roundTripTest -g80000000 -P99 "19 -T2" " "
roundTripTest -g5000000000 -P99 "1 -T2" " " roundTripTest -g5000000000 -P99 "1 -T2" " "
roundTripTest -g500000000 -P97 "1 -T999" " " roundTripTest -g500000000 -P97 "1 -T999" " "

14
tests/zstreamtest.c
View File

@ -254,9 +254,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) ); CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
outBuff.dst = (char*)(compressedBuffer)+cSize; outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize; assert(compressedBufferSize > cSize);
outBuff.size = compressedBufferSize - cSize;
outBuff.pos = 0; outBuff.pos = 0;
inBuff.src = CNBuffer; inBuff.src = CNBuffer;
inBuff.size = CNBufferSize; inBuff.size = CNBufferSize;
@ -622,7 +623,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) ); CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
} /* cstream advanced shall write content size = 0 */ } /* cstream advanced shall write content size = 0 */
inBuff.src = CNBuffer; inBuff.src = CNBuffer;
inBuff.size = 0; inBuff.size = 0;
@ -935,10 +936,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart; dict = srcBuffer + dictStart;
} }
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} } } }
@ -1189,13 +1191,13 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart; dict = srcBuffer + dictStart;
} }
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n", DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
params.cParams.windowLog, (U32)pledgedSrcSize); params.cParams.windowLog, (U32)pledgedSrcSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
params.fParams.contentSizeFlag = pledgedSrcSize>0; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );