facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1444 from facebook/btultra2 

btultra2
dev
Yann Collet 2018-12-12 21:09:01 -08:00 committed by GitHub
parent 54ca4b3c5d 9a92ed401d
commit fbcae274a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 855 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -64,7 +64,8 @@ zlibwrapper: lib
## test: run long-duration tests ## test: run long-duration tests
.PHONY: test .PHONY: test
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror DEBUGLEVEL ?= 1
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
test: test:
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@ $(MAKE) -C $(TESTDIR) $@

10
doc/zstd_manual.html
View File

@ -399,10 +399,10 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_lazy2=5, ZSTD_lazy2=5,
ZSTD_btlazy2=6, ZSTD_btlazy2=6,
ZSTD_btopt=7, ZSTD_btopt=7,
ZSTD_btultra=8 ZSTD_btultra=8,
</b>/* note : new strategies might be added in the future.<b> ZSTD_btultra2=9
Only the order (from fast to strong) is guaranteed, not the exact position. </b>/* note : new strategies _might_ be added in the future.<b>
new strategy names might be introduced, pushing the maximum number upward */ Only the order (from fast to strong) is guaranteed */
} ZSTD_strategy; } ZSTD_strategy;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
@ -452,7 +452,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Distance between match sampling. * Distance between match sampling.
* Larger values make compression faster, and weaker. * Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */ * Special: value 0 means "use default targetLength". */
ZSTD_c_compressionStrategy=107, </b>/* See ZSTD_strategy enum definition.<b> ZSTD_c_strategy=107, </b>/* See ZSTD_strategy enum definition.<b>
* The higher the value of selected strategy, the more complex it is, * The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression. * resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */ * Special: value 0 means "use default strategy". */

287
lib/compress/zstd_compress.c
View File

@ -267,9 +267,9 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = ZSTD_TARGETLENGTH_MAX; bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
return bounds; return bounds;
case ZSTD_c_compressionStrategy: case ZSTD_c_strategy:
bounds.lowerBound = (int)ZSTD_fast; bounds.lowerBound = ZSTD_STRATEGY_MIN;
bounds.upperBound = (int)ZSTD_btultra; /* note : how to ensure at compile time that this is the highest value strategy ? */ bounds.upperBound = ZSTD_STRATEGY_MAX;
return bounds; return bounds;
case ZSTD_c_contentSizeFlag: case ZSTD_c_contentSizeFlag:
@ -347,14 +347,15 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
return bounds; return bounds;
case ZSTD_c_format: case ZSTD_c_format:
ZSTD_STATIC_ASSERT((int)ZSTD_f_zstd1 < (int)ZSTD_f_zstd1_magicless); ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
bounds.lowerBound = (int)ZSTD_f_zstd1; bounds.lowerBound = ZSTD_f_zstd1;
bounds.upperBound = (int)ZSTD_f_zstd1_magicless; bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds; return bounds;
case ZSTD_c_forceAttachDict: case ZSTD_c_forceAttachDict:
bounds.lowerBound = 0; ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy);
bounds.upperBound = 1; bounds.lowerBound = ZSTD_dictDefaultAttach;
bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds; return bounds;
default: default:
@ -364,9 +365,21 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
} }
} }
#define CLAMPCHECK(val,min,max) { \ /* ZSTD_cParam_withinBounds:
if (((val)<(min)) | ((val)>(max))) { \ * @return 1 if value is within cParam bounds,
return ERROR(parameter_outOfBound); \ * 0 otherwise */
static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
{
ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
if (ZSTD_isError(bounds.error)) return 0;
if (value < bounds.lowerBound) return 0;
if (value > bounds.upperBound) return 0;
return 1;
}
#define BOUNDCHECK(cParam, val) { \
if (!ZSTD_cParam_withinBounds(cParam,val)) { \
return ERROR(parameter_outOfBound); \
} } } }
@ -380,7 +393,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_searchLog: case ZSTD_c_searchLog:
case ZSTD_c_minMatch: case ZSTD_c_minMatch:
case ZSTD_c_targetLength: case ZSTD_c_targetLength:
case ZSTD_c_compressionStrategy: case ZSTD_c_strategy:
return 1; return 1;
case ZSTD_c_format: case ZSTD_c_format:
@ -429,7 +442,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_searchLog: case ZSTD_c_searchLog:
case ZSTD_c_minMatch: case ZSTD_c_minMatch:
case ZSTD_c_targetLength: case ZSTD_c_targetLength:
case ZSTD_c_compressionStrategy: case ZSTD_c_strategy:
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@ -476,14 +489,14 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
switch(param) switch(param)
{ {
case ZSTD_c_format : case ZSTD_c_format :
if (value > (int)ZSTD_f_zstd1_magicless) BOUNDCHECK(ZSTD_c_format, value);
return ERROR(parameter_unsupported);
CCtxParams->format = (ZSTD_format_e)value; CCtxParams->format = (ZSTD_format_e)value;
return (size_t)CCtxParams->format; return (size_t)CCtxParams->format;
case ZSTD_c_compressionLevel : { case ZSTD_c_compressionLevel : {
int cLevel = value; int cLevel = value;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel();
if (cLevel) { /* 0 : does not change current level */ if (cLevel) { /* 0 : does not change current level */
CCtxParams->compressionLevel = cLevel; CCtxParams->compressionLevel = cLevel;
} }
@ -493,42 +506,42 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_windowLog : case ZSTD_c_windowLog :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); BOUNDCHECK(ZSTD_c_windowLog, value);
CCtxParams->cParams.windowLog = value; CCtxParams->cParams.windowLog = value;
return CCtxParams->cParams.windowLog; return CCtxParams->cParams.windowLog;
case ZSTD_c_hashLog : case ZSTD_c_hashLog :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); BOUNDCHECK(ZSTD_c_hashLog, value);
CCtxParams->cParams.hashLog = value; CCtxParams->cParams.hashLog = value;
return CCtxParams->cParams.hashLog; return CCtxParams->cParams.hashLog;
case ZSTD_c_chainLog : case ZSTD_c_chainLog :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); BOUNDCHECK(ZSTD_c_chainLog, value);
CCtxParams->cParams.chainLog = value; CCtxParams->cParams.chainLog = value;
return CCtxParams->cParams.chainLog; return CCtxParams->cParams.chainLog;
case ZSTD_c_searchLog : case ZSTD_c_searchLog :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); BOUNDCHECK(ZSTD_c_searchLog, value);
CCtxParams->cParams.searchLog = value; CCtxParams->cParams.searchLog = value;
return value; return value;
case ZSTD_c_minMatch : case ZSTD_c_minMatch :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); BOUNDCHECK(ZSTD_c_minMatch, value);
CCtxParams->cParams.minMatch = value; CCtxParams->cParams.minMatch = value;
return CCtxParams->cParams.minMatch; return CCtxParams->cParams.minMatch;
case ZSTD_c_targetLength : case ZSTD_c_targetLength :
/* all values are valid. 0 => use default */ BOUNDCHECK(ZSTD_c_targetLength, value);
CCtxParams->cParams.targetLength = value; CCtxParams->cParams.targetLength = value;
return CCtxParams->cParams.targetLength; return CCtxParams->cParams.targetLength;
case ZSTD_c_compressionStrategy : case ZSTD_c_strategy :
if (value!=0) /* 0 => use default */ if (value!=0) /* 0 => use default */
CLAMPCHECK(value, (int)ZSTD_fast, (int)ZSTD_btultra); BOUNDCHECK(ZSTD_c_strategy, value);
CCtxParams->cParams.strategy = (ZSTD_strategy)value; CCtxParams->cParams.strategy = (ZSTD_strategy)value;
return (size_t)CCtxParams->cParams.strategy; return (size_t)CCtxParams->cParams.strategy;
@ -554,7 +567,7 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_forceAttachDict : { case ZSTD_c_forceAttachDict : {
const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
CLAMPCHECK(pref, ZSTD_dictDefaultAttach, ZSTD_dictForceCopy); BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
CCtxParams->attachDictPref = pref; CCtxParams->attachDictPref = pref;
return CCtxParams->attachDictPref; return CCtxParams->attachDictPref;
} }
@ -594,19 +607,19 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_ldmHashLog : case ZSTD_c_ldmHashLog :
if (value!=0) /* 0 ==> auto */ if (value!=0) /* 0 ==> auto */
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); BOUNDCHECK(ZSTD_c_ldmHashLog, value);
CCtxParams->ldmParams.hashLog = value; CCtxParams->ldmParams.hashLog = value;
return CCtxParams->ldmParams.hashLog; return CCtxParams->ldmParams.hashLog;
case ZSTD_c_ldmMinMatch : case ZSTD_c_ldmMinMatch :
if (value!=0) /* 0 ==> default */ if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
CCtxParams->ldmParams.minMatchLength = value; CCtxParams->ldmParams.minMatchLength = value;
return CCtxParams->ldmParams.minMatchLength; return CCtxParams->ldmParams.minMatchLength;
case ZSTD_c_ldmBucketSizeLog : case ZSTD_c_ldmBucketSizeLog :
if (value!=0) /* 0 ==> default */ if (value!=0) /* 0 ==> default */
CLAMPCHECK(value, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX); BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
CCtxParams->ldmParams.bucketSizeLog = value; CCtxParams->ldmParams.bucketSizeLog = value;
return CCtxParams->ldmParams.bucketSizeLog; return CCtxParams->ldmParams.bucketSizeLog;
@ -654,7 +667,7 @@ size_t ZSTD_CCtxParam_getParameter(
case ZSTD_c_targetLength : case ZSTD_c_targetLength :
*value = CCtxParams->cParams.targetLength; *value = CCtxParams->cParams.targetLength;
break; break;
case ZSTD_c_compressionStrategy : case ZSTD_c_strategy :
*value = (unsigned)CCtxParams->cParams.strategy; *value = (unsigned)CCtxParams->cParams.strategy;
break; break;
case ZSTD_c_contentSizeFlag : case ZSTD_c_contentSizeFlag :
@ -832,16 +845,13 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
@return : 0, or an error code if one value is beyond authorized range */ @return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{ {
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog);
CLAMPCHECK(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0); BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX) BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
return ERROR(parameter_outOfBound);
if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
return ERROR(parameter_unsupported);
return 0; return 0;
} }
@ -851,19 +861,19 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
static ZSTD_compressionParameters static ZSTD_compressionParameters
ZSTD_clampCParams(ZSTD_compressionParameters cParams) ZSTD_clampCParams(ZSTD_compressionParameters cParams)
{ {
# define CLAMP(val,min,max) { \ # define CLAMP_TYPE(cParam, val, type) { \
if (val<min) val=min; \ ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \
else if (val>max) val=max; \ if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
} }
CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); # define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int)
CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMP(ZSTD_c_windowLog, cParams.windowLog);
CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMP(ZSTD_c_chainLog, cParams.chainLog);
CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMP(ZSTD_c_hashLog, cParams.hashLog);
CLAMP(cParams.minMatch, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); CLAMP(ZSTD_c_searchLog, cParams.searchLog);
ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0); CLAMP(ZSTD_c_minMatch, cParams.minMatch);
if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX) CLAMP(ZSTD_c_targetLength,cParams.targetLength);
cParams.targetLength = ZSTD_TARGETLENGTH_MAX; CLAMP_TYPE(ZSTD_c_strategy, cParams.strategy, ZSTD_strategy);
CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
return cParams; return cParams;
} }
@ -951,8 +961,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32) size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32)
+ (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t)); + (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t));
size_t const optSpace = (forCCtx && ((cParams->strategy == ZSTD_btopt) || size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
(cParams->strategy == ZSTD_btultra)))
? optPotentialSpace ? optPotentialSpace
: 0; : 0;
DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
@ -1253,7 +1262,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ZSTD_invalidateMatchState(ms); ZSTD_invalidateMatchState(ms);
/* opt parser space */ /* opt parser space */
if (forCCtx && ((cParams->strategy == ZSTD_btopt) | (cParams->strategy == ZSTD_btultra))) { if (forCCtx && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space"); DEBUGLOG(4, "reserving optimal parser space");
ms->opt.litFreq = (U32*)ptr; ms->opt.litFreq = (U32*)ptr;
ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits); ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits);
@ -1465,16 +1474,17 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
* dictionary tables into the working context is faster than using them * dictionary tables into the working context is faster than using them
* in-place. * in-place.
*/ */
static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = { static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = {
8 KB, /* unused */ 8 KB, /* unused */
8 KB, /* ZSTD_fast */ 8 KB, /* ZSTD_fast */
16 KB, /* ZSTD_dfast */ 16 KB, /* ZSTD_dfast */
32 KB, /* ZSTD_greedy */ 32 KB, /* ZSTD_greedy */
32 KB, /* ZSTD_lazy */ 32 KB, /* ZSTD_lazy */
32 KB, /* ZSTD_lazy2 */ 32 KB, /* ZSTD_lazy2 */
32 KB, /* ZSTD_btlazy2 */ 32 KB, /* ZSTD_btlazy2 */
32 KB, /* ZSTD_btopt */ 32 KB, /* ZSTD_btopt */
8 KB /* ZSTD_btultra */ 8 KB, /* ZSTD_btultra */
8 KB /* ZSTD_btultra2 */
}; };
static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
@ -1829,7 +1839,9 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
* note : use same formula for both situations */ * note : use same formula for both situations */
static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
{ {
U32 const minlog = (strat==ZSTD_btultra) ? 7 : 6; U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
return (srcSize >> minlog) + 2; return (srcSize >> minlog) + 2;
} }
@ -2423,7 +2435,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table"); DEBUGLOG(5, "Building LL table");
nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, count, max, mostFrequent, nbSeq, LLFSELog, prevEntropy->fse.litlengthCTable, LL_defaultNorm, LL_defaultNormLog, ZSTD_defaultAllowed, strategy); LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode,
count, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->fse.litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed); assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
@ -2442,7 +2458,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table"); DEBUGLOG(5, "Building OF table");
nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode;
Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, count, max, mostFrequent, nbSeq, OffFSELog, prevEntropy->fse.offcodeCTable, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode,
count, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->fse.offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
@ -2458,7 +2478,11 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, count, max, mostFrequent, nbSeq, MLFSELog, prevEntropy->fse.matchlengthCTable, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode,
count, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->fse.matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
@ -2536,7 +2560,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
* assumption : strat is a valid strategy */ * assumption : strat is a valid strategy */
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode)
{ {
static const ZSTD_blockCompressor blockCompressor[3][(unsigned)ZSTD_btultra+1] = { static const ZSTD_blockCompressor blockCompressor[3][ZSTD_STRATEGY_MAX+1] = {
{ ZSTD_compressBlock_fast /* default for 0 */, { ZSTD_compressBlock_fast /* default for 0 */,
ZSTD_compressBlock_fast, ZSTD_compressBlock_fast,
ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_doubleFast,
@ -2545,7 +2569,8 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2, ZSTD_compressBlock_lazy2,
ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btlazy2,
ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt,
ZSTD_compressBlock_btultra }, ZSTD_compressBlock_btultra,
ZSTD_compressBlock_btultra2 },
{ ZSTD_compressBlock_fast_extDict /* default for 0 */, { ZSTD_compressBlock_fast_extDict /* default for 0 */,
ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_doubleFast_extDict,
@ -2554,6 +2579,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_lazy2_extDict,
ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btlazy2_extDict,
ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt_extDict,
ZSTD_compressBlock_btultra_extDict,
ZSTD_compressBlock_btultra_extDict }, ZSTD_compressBlock_btultra_extDict },
{ ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */,
ZSTD_compressBlock_fast_dictMatchState, ZSTD_compressBlock_fast_dictMatchState,
@ -2563,14 +2589,14 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_compressBlock_lazy2_dictMatchState, ZSTD_compressBlock_lazy2_dictMatchState,
ZSTD_compressBlock_btlazy2_dictMatchState, ZSTD_compressBlock_btlazy2_dictMatchState,
ZSTD_compressBlock_btopt_dictMatchState, ZSTD_compressBlock_btopt_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState } ZSTD_compressBlock_btultra_dictMatchState }
}; };
ZSTD_blockCompressor selectedCompressor; ZSTD_blockCompressor selectedCompressor;
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert((U32)strat >= (U32)ZSTD_fast); assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
assert((U32)strat <= (U32)ZSTD_btultra); selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
selectedCompressor = blockCompressor[(int)dictMode][(U32)strat];
assert(selectedCompressor != NULL); assert(selectedCompressor != NULL);
return selectedCompressor; return selectedCompressor;
} }
@ -2967,6 +2993,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btultra: case ZSTD_btultra:
case ZSTD_btultra2:
if (srcSize >= HASH_READ_SIZE) if (srcSize >= HASH_READ_SIZE)
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
break; break;
@ -3037,7 +3064,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
/* fill all offset symbols to avoid garbage at end of table */ /* fill all offset symbols to avoid garbage at end of table */
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE), CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable,
offcodeNCount, MaxOff, offcodeLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted); dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
@ -3049,7 +3078,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */ /* Every match length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE), CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable,
matchlengthNCount, matchlengthMaxValue, matchlengthLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted); dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
@ -3061,7 +3092,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */ /* Every literal length code must have non-zero probability */
CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE), CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable,
litlengthNCount, litlengthMaxValue, litlengthLog,
workspace, HUF_WORKSPACE_SIZE),
dictionary_corrupted); dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
@ -4115,27 +4148,27 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
/* W, C, H, S, L, TL, strat */ /* W, C, H, S, L, TL, strat */
{ 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
{ 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
{ 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ { 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */
{ 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ { 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */
{ 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */ { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */
{ 21, 18, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */ { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */
{ 21, 18, 19, 3, 5, 8, ZSTD_lazy2 }, /* level 7 */ { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */
{ 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */
{ 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
{ 21, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
{ 21, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */
{ 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
{ 22, 22, 22, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
{ 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
{ 23, 22, 22, 4, 4, 64, ZSTD_btopt }, /* level 17 */ { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 3,256, ZSTD_btopt }, /* level 18 */ { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
{ 23, 24, 22, 7, 3,256, ZSTD_btultra }, /* level 19 */ { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
{ 25, 25, 23, 7, 3,256, ZSTD_btultra }, /* level 20 */ { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
{ 26, 26, 24, 7, 3,512, ZSTD_btultra }, /* level 21 */ { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
{ 27, 27, 25, 9, 3,999, ZSTD_btultra }, /* level 22 */ { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
}, },
{ /* for srcSize <= 256 KB */ { /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
@ -4150,18 +4183,18 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 18, 18, 19, 5, 4, 16, ZSTD_btlazy2 }, /* level 11.*/ { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
{ 18, 19, 19, 6, 4, 16, ZSTD_btlazy2 }, /* level 12.*/ { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
{ 18, 19, 19, 8, 4, 16, ZSTD_btlazy2 }, /* level 13 */ { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
{ 18, 18, 19, 4, 4, 24, ZSTD_btopt }, /* level 14.*/ { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
{ 18, 18, 19, 4, 3, 24, ZSTD_btopt }, /* level 15.*/ { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 19, 6, 3, 64, ZSTD_btopt }, /* level 16.*/ { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
{ 18, 19, 19, 8, 3,128, ZSTD_btopt }, /* level 17.*/ { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
{ 18, 19, 19, 10, 3,256, ZSTD_btopt }, /* level 18.*/ { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
{ 18, 19, 19, 10, 3,256, ZSTD_btultra }, /* level 19.*/ { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 18, 19, 19, 11, 3,512, ZSTD_btultra }, /* level 20.*/ { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 18, 19, 19, 12, 3,512, ZSTD_btultra }, /* level 21.*/ { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 18, 19, 19, 13, 3,999, ZSTD_btultra }, /* level 22.*/ { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
}, },
{ /* for srcSize <= 128 KB */ { /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
@ -4176,26 +4209,26 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
{ 17, 18, 17, 6, 4, 16, ZSTD_btlazy2 }, /* level 12 */ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
{ 17, 18, 17, 8, 4, 16, ZSTD_btlazy2 }, /* level 13.*/ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
{ 17, 18, 17, 4, 4, 32, ZSTD_btopt }, /* level 14.*/ { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
{ 17, 18, 17, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
{ 17, 18, 17, 7, 3,128, ZSTD_btopt }, /* level 16.*/ { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 17.*/ { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 18.*/ { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 19.*/ { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
}, },
{ /* for srcSize <= 16 KB */ { /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
{ 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
{ 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/ { 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
{ 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
{ 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
{ 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
@ -4203,17 +4236,17 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
{ 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
{ 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
{ 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
{ 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
{ 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
{ 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
{ 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 18.*/ { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/ { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
{ 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
{ 14, 15, 15, 10, 3,512, ZSTD_btultra }, /* level 22.*/ { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
}, },
}; };
@ -4232,8 +4265,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
{ ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */
return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); } return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);
}
} }
/*! ZSTD_getParams() : /*! ZSTD_getParams() :

1
lib/compress/zstd_ldm.c
View File

@ -188,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
case ZSTD_btlazy2: case ZSTD_btlazy2:
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btultra: case ZSTD_btultra:
case ZSTD_btultra2:
break; break;
default: default:
assert(0); /* not possible : not a valid strategy id */ assert(0); /* not possible : not a valid strategy id */

74
lib/compress/zstd_opt.c
View File

@ -1071,39 +1071,69 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1); optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1); optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
} }
/* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function must not fail, hence its usage conditions must be respected.
*/
static void ZSTD_initStats_ultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
DEBUGLOG(5, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
}
size_t ZSTD_compressBlock_btultra( size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
#if 0 return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
/* 2-pass strategy (disabled) }
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
/* 2-pass strategy
* this strategy makes a first pass over first block to collect statistics * this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it. * and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.5% on first block), * The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */ * the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX); assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */ if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
U32 tmpRep[ZSTD_REP_NUM]; && (ms->window.dictLimit - ms->nextToUpdate <= 1) /* no prefix (note: intentional overflow, defined as 2-complement) */
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics"); ) {
assert(ms->nextToUpdate >= ms->window.dictLimit ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
ZSTD_resetSeqStore(seqStore);
/* invalidate first scan from history */
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
} }
#endif
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
} }
@ -1134,3 +1164,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
{ {
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
} }
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */

8
lib/compress/zstd_opt.h
View File

@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt(
size_t ZSTD_compressBlock_btultra( size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize); void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState( size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize); void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

15
lib/zstd.h
View File

@ -494,10 +494,10 @@ typedef enum { ZSTD_fast=1,
ZSTD_lazy2=5, ZSTD_lazy2=5,
ZSTD_btlazy2=6, ZSTD_btlazy2=6,
ZSTD_btopt=7, ZSTD_btopt=7,
ZSTD_btultra=8 ZSTD_btultra=8,
/* note : new strategies might be added in the future. ZSTD_btultra2=9
Only the order (from fast to strong) is guaranteed, not the exact position. /* note : new strategies _might_ be added in the future.
new strategy names might be introduced, pushing the maximum number upward */ Only the order (from fast to strong) is guaranteed */
} ZSTD_strategy; } ZSTD_strategy;
@ -541,14 +541,14 @@ typedef enum {
* , for all strategies > fast, effective maximum is 6. * , for all strategies > fast, effective maximum is 6.
* Special: value 0 means "use default minMatchLength". */ * Special: value 0 means "use default minMatchLength". */
ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
* For strategies btopt & btultra: * For strategies btopt, btultra & btultra2:
* Length of Match considered "good enough" to stop search. * Length of Match considered "good enough" to stop search.
* Larger values make compression stronger, and slower. * Larger values make compression stronger, and slower.
* For strategy fast: * For strategy fast:
* Distance between match sampling. * Distance between match sampling.
* Larger values make compression faster, and weaker. * Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */ * Special: value 0 means "use default targetLength". */
ZSTD_c_compressionStrategy=107, /* See ZSTD_strategy enum definition. ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
* The higher the value of selected strategy, the more complex it is, * The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression. * resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */ * Special: value 0 means "use default strategy". */
@ -947,6 +947,9 @@ ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ #define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
#define ZSTD_STRATEGY_MIN ZSTD_fast
#define ZSTD_STRATEGY_MAX ZSTD_btultra2
#define ZSTD_OVERLAPLOG_MIN 0 #define ZSTD_OVERLAPLOG_MIN 0
#define ZSTD_OVERLAPLOG_MAX 9 #define ZSTD_OVERLAPLOG_MAX 9

59
programs/benchzstd.c
View File

@ -94,6 +94,18 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return errorNum; \ return errorNum; \
} }
#define CHECK_Z(zf) { \
size_t const zerr = zf; \
if (ZSTD_isError(zerr)) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAY("Error : "); \
DISPLAY("%s failed : %s", \
#zf, ZSTD_getErrorName(zerr)); \
DISPLAY(" \n"); \
exit(1); \
} \
}
#define RETURN_ERROR(errorNum, retType, ...) { \ #define RETURN_ERROR(errorNum, retType, ...) { \
retType r; \ retType r; \
memset(&r, 0, sizeof(retType)); \ memset(&r, 0, sizeof(retType)); \
@ -105,17 +117,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return r; \ return r; \
} }
/* error without displaying */
#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.tag = errorNum; \
return r; \
}
/* ************************************* /* *************************************
* Benchmark Parameters * Benchmark Parameters
@ -162,30 +163,30 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) { if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
} else { } else {
ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
} }
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionStrategy, comprParams->strategy); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
} }
static void BMK_initDCtx(ZSTD_DCtx* dctx, static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) { const void* dictBuffer, size_t dictBufferSize) {
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
} }

2
programs/fileio.c
View File

@ -551,7 +551,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionStrategy, comprParams.strategy) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */ /* multi-threading */
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers); DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);

6
programs/zstd.1
View File

@ -1,5 +1,5 @@
. .
.TH "ZSTD" "1" "November 2018" "zstd 1.3.8" "User Commands" .TH "ZSTD" "1" "December 2018" "zstd 1.3.8" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -316,7 +316,7 @@ set process priority to real\-time
Specify a strategy used by a match finder\. Specify a strategy used by a match finder\.
. .
.IP .IP
There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra\. There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
. .
.TP .TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR \fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@ -373,7 +373,7 @@ The minimum \fImml\fR is 3 and the maximum is 7\.
The impact of this field vary depending on selected strategy\. The impact of this field vary depending on selected strategy\.
. .
.IP .IP
For ZSTD_btopt and ZSTD_btultra, it specifies the minimum match length that causes match finder to stop searching for better matches\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\. For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
. .
.IP .IP
For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\. For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.

11
programs/zstd.1.md
View File

@ -339,9 +339,10 @@ The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_: - `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder. Specify a strategy used by a match finder.
There are 8 strategies numbered from 1 to 8, from faster to stronger: There are 9 strategies numbered from 1 to 9, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy, 1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra. 4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
- `windowLog`=_wlog_, `wlog`=_wlog_: - `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance. Specify the maximum number of bits for a match distance.
@ -394,8 +395,8 @@ The list of available _options_:
- `targetLen`=_tlen_, `tlen`=_tlen_: - `targetLen`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy. The impact of this field vary depending on selected strategy.
For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
that causes match finder to stop searching for better matches. the minimum match length that causes match finder to stop searching.
A larger `targetLen` usually improves compression ratio A larger `targetLen` usually improves compression ratio
but decreases compression speed. but decreases compression speed.

2
programs/zstdgrep.1
View File

@ -1,5 +1,5 @@
. .
.TH "ZSTDGREP" "1" "November 2018" "zstd 1.3.8" "User Commands" .TH "ZSTDGREP" "1" "December 2018" "zstd 1.3.8" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files

2
programs/zstdless.1
View File

@ -1,5 +1,5 @@
. .
.TH "ZSTDLESS" "1" "November 2018" "zstd 1.3.8" "User Commands" .TH "ZSTDLESS" "1" "December 2018" "zstd 1.3.8" "User Commands"
. .
.SH "NAME" .SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files \fBzstdless\fR \- view zstandard\-compressed files

4
tests/fullbench.c
View File

@ -401,7 +401,7 @@ static size_t benchMem(U32 benchNb,
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog); ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch); ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength); ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionStrategy, cparams.strategy); ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
@ -411,7 +411,7 @@ static size_t benchMem(U32 benchNb,
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionStrategy, cparams.strategy); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
/* Preparation */ /* Preparation */
switch(benchNb) switch(benchNb)

6
tests/fuzz/zstd_helpers.c
View File

@ -13,7 +13,7 @@
#include "fuzz_helpers.h" #include "fuzz_helpers.h"
#include "zstd.h" #include "zstd.h"
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value) static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
{ {
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
} }
@ -35,7 +35,7 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN, cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
ZSTD_MINMATCH_MAX); ZSTD_MINMATCH_MAX);
cParams.targetLength = FUZZ_rand32(state, 0, 512); cParams.targetLength = FUZZ_rand32(state, 0, 512);
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra); cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
return ZSTD_adjustCParams(cParams, srcSize, 0); return ZSTD_adjustCParams(cParams, srcSize, 0);
} }
@ -66,7 +66,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
set(cctx, ZSTD_c_searchLog, cParams.searchLog); set(cctx, ZSTD_c_searchLog, cParams.searchLog);
set(cctx, ZSTD_c_minMatch, cParams.minMatch); set(cctx, ZSTD_c_minMatch, cParams.minMatch);
set(cctx, ZSTD_c_targetLength, cParams.targetLength); set(cctx, ZSTD_c_targetLength, cParams.targetLength);
set(cctx, ZSTD_c_compressionStrategy, cParams.strategy); set(cctx, ZSTD_c_strategy, cParams.strategy);
/* Select frame parameters */ /* Select frame parameters */
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);

670
tests/paramgrill.c
View File

File diff suppressed because it is too large Load Diff

3
tests/playTests.sh
View File

@ -237,6 +237,7 @@ $ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
test ! -f tmp.zst # tmp.zst should not be created test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K roundTripTest -g512K
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
@ -244,7 +245,7 @@ roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g512K 19 roundTripTest -g64K "19 --zstd=strat=9" # btultra2
$ECHO "\n===> Pass-Through mode " $ECHO "\n===> Pass-Through mode "

3
tests/regression/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# regression test artifacts
data-cache
test

10
tests/regression/config.c
View File

@ -77,10 +77,11 @@ int config_skip_data(config_t const* config, data_t const* data) {
return config->use_dictionary && !data_has_dict(data); return config->use_dictionary && !data_has_dict(data);
} }
int config_get_level(config_t const* config) { int config_get_level(config_t const* config)
{
param_values_t const params = config->param_values; param_values_t const params = config->param_values;
size_t i; size_t i;
for (size_t i = 0; i < params.size; ++i) { for (i = 0; i < params.size; ++i) {
if (params.data[i].param == ZSTD_c_compressionLevel) if (params.data[i].param == ZSTD_c_compressionLevel)
return (int)params.data[i].value; return (int)params.data[i].value;
} }
@ -90,7 +91,8 @@ int config_get_level(config_t const* config) {
ZSTD_parameters config_get_zstd_params( ZSTD_parameters config_get_zstd_params(
config_t const* config, config_t const* config,
uint64_t srcSize, uint64_t srcSize,
size_t dictSize) { size_t dictSize)
{
ZSTD_parameters zparams = {}; ZSTD_parameters zparams = {};
param_values_t const params = config->param_values; param_values_t const params = config->param_values;
int level = config_get_level(config); int level = config_get_level(config);
@ -130,7 +132,7 @@ ZSTD_parameters config_get_zstd_params(
case ZSTD_c_targetLength: case ZSTD_c_targetLength:
zparams.cParams.targetLength = value; zparams.cParams.targetLength = value;
break; break;
case ZSTD_c_compressionStrategy: case ZSTD_c_strategy:
zparams.cParams.strategy = (ZSTD_strategy)value; zparams.cParams.strategy = (ZSTD_strategy)value;
break; break;
default: default:

250
tests/regression/results.csv
View File

@ -2,17 +2,17 @@ Data, Config, Method,
silesia.tar, level -5, compress simple, 7160438 silesia.tar, level -5, compress simple, 7160438
silesia.tar, level -3, compress simple, 6789024 silesia.tar, level -3, compress simple, 6789024
silesia.tar, level -1, compress simple, 6195462 silesia.tar, level -1, compress simple, 6195462
silesia.tar, level 0, compress simple, 4875071 silesia.tar, level 0, compress simple, 4875008
silesia.tar, level 1, compress simple, 5339697 silesia.tar, level 1, compress simple, 5339697
silesia.tar, level 3, compress simple, 4875071 silesia.tar, level 3, compress simple, 4875008
silesia.tar, level 4, compress simple, 4813104 silesia.tar, level 4, compress simple, 4813507
silesia.tar, level 5, compress simple, 4726961 silesia.tar, level 5, compress simple, 4722235
silesia.tar, level 6, compress simple, 4654401 silesia.tar, level 6, compress simple, 4672194
silesia.tar, level 7, compress simple, 4591933 silesia.tar, level 7, compress simple, 4606658
silesia.tar, level 9, compress simple, 4554098 silesia.tar, level 9, compress simple, 4554098
silesia.tar, level 13, compress simple, 4503496 silesia.tar, level 13, compress simple, 4491702
silesia.tar, level 16, compress simple, 4387233 silesia.tar, level 16, compress simple, 4381277
silesia.tar, level 19, compress simple, 4283123 silesia.tar, level 19, compress simple, 4281581
silesia, level -5, compress cctx, 7152294 silesia, level -5, compress cctx, 7152294
silesia, level -3, compress cctx, 6789969 silesia, level -3, compress cctx, 6789969
silesia, level -1, compress cctx, 6191548 silesia, level -1, compress cctx, 6191548
@ -20,13 +20,13 @@ silesia, level 0, compress cctx,
silesia, level 1, compress cctx, 5318036 silesia, level 1, compress cctx, 5318036
silesia, level 3, compress cctx, 4862377 silesia, level 3, compress cctx, 4862377
silesia, level 4, compress cctx, 4800629 silesia, level 4, compress cctx, 4800629
silesia, level 5, compress cctx, 4715005 silesia, level 5, compress cctx, 4710178
silesia, level 6, compress cctx, 4644055 silesia, level 6, compress cctx, 4659996
silesia, level 7, compress cctx, 4581559 silesia, level 7, compress cctx, 4596234
silesia, level 9, compress cctx, 4543862 silesia, level 9, compress cctx, 4543862
silesia, level 13, compress cctx, 4493931 silesia, level 13, compress cctx, 4482073
silesia, level 16, compress cctx, 4381885 silesia, level 16, compress cctx, 4377391
silesia, level 19, compress cctx, 4296899 silesia, level 19, compress cctx, 4294841
github, level -5, compress cctx, 232744 github, level -5, compress cctx, 232744
github, level -5 with dict, compress cctx, 45704 github, level -5 with dict, compress cctx, 45704
github, level -3, compress cctx, 220611 github, level -3, compress cctx, 220611
@ -49,11 +49,11 @@ github, level 7, compress cctx,
github, level 7 with dict, compress cctx, 38741 github, level 7 with dict, compress cctx, 38741
github, level 9, compress cctx, 135108 github, level 9, compress cctx, 135108
github, level 9 with dict, compress cctx, 39335 github, level 9 with dict, compress cctx, 39335
github, level 13, compress cctx, 133741 github, level 13, compress cctx, 133717
github, level 13 with dict, compress cctx, 39670 github, level 13 with dict, compress cctx, 39923
github, level 16, compress cctx, 133741 github, level 16, compress cctx, 134844
github, level 16 with dict, compress cctx, 37928 github, level 16 with dict, compress cctx, 37568
github, level 19, compress cctx, 133717 github, level 19, compress cctx, 134675
github, level 19 with dict, compress cctx, 37567 github, level 19 with dict, compress cctx, 37567
silesia, level -5, zstdcli, 7152342 silesia, level -5, zstdcli, 7152342
silesia, level -3, zstdcli, 6790021 silesia, level -3, zstdcli, 6790021
@ -62,28 +62,28 @@ silesia, level 0, zstdcli,
silesia, level 1, zstdcli, 5318084 silesia, level 1, zstdcli, 5318084
silesia, level 3, zstdcli, 4862425 silesia, level 3, zstdcli, 4862425
silesia, level 4, zstdcli, 4800677 silesia, level 4, zstdcli, 4800677
silesia, level 5, zstdcli, 4715053 silesia, level 5, zstdcli, 4710226
silesia, level 6, zstdcli, 4644103 silesia, level 6, zstdcli, 4660044
silesia, level 7, zstdcli, 4581607 silesia, level 7, zstdcli, 4596282
silesia, level 9, zstdcli, 4543910 silesia, level 9, zstdcli, 4543910
silesia, level 13, zstdcli, 4493979 silesia, level 13, zstdcli, 4482121
silesia, level 16, zstdcli, 4381933 silesia, level 16, zstdcli, 4377439
silesia, level 19, zstdcli, 4296947 silesia, level 19, zstdcli, 4294889
silesia.tar, level -5, zstdcli, 7159586 silesia.tar, level -5, zstdcli, 7159586
silesia.tar, level -3, zstdcli, 6791018 silesia.tar, level -3, zstdcli, 6791018
silesia.tar, level -1, zstdcli, 6196283 silesia.tar, level -1, zstdcli, 6196283
silesia.tar, level 0, zstdcli, 4876730 silesia.tar, level 0, zstdcli, 4875213
silesia.tar, level 1, zstdcli, 5340312 silesia.tar, level 1, zstdcli, 5340312
silesia.tar, level 3, zstdcli, 4876730 silesia.tar, level 3, zstdcli, 4875213
silesia.tar, level 4, zstdcli, 4817723 silesia.tar, level 4, zstdcli, 4814545
silesia.tar, level 5, zstdcli, 4730389 silesia.tar, level 5, zstdcli, 4723284
silesia.tar, level 6, zstdcli, 4655708 silesia.tar, level 6, zstdcli, 4673591
silesia.tar, level 7, zstdcli, 4593407 silesia.tar, level 7, zstdcli, 4608342
silesia.tar, level 9, zstdcli, 4556135 silesia.tar, level 9, zstdcli, 4556135
silesia.tar, level 13, zstdcli, 4503500 silesia.tar, level 13, zstdcli, 4491706
silesia.tar, level 16, zstdcli, 4387237 silesia.tar, level 16, zstdcli, 4381281
silesia.tar, level 19, zstdcli, 4283127 silesia.tar, level 19, zstdcli, 4281585
silesia.tar, no source size, zstdcli, 4876726 silesia.tar, no source size, zstdcli, 4875209
github, level -5, zstdcli, 234744 github, level -5, zstdcli, 234744
github, level -5 with dict, zstdcli, 47528 github, level -5 with dict, zstdcli, 47528
github, level -3, zstdcli, 222611 github, level -3, zstdcli, 222611
@ -106,11 +106,11 @@ github, level 7, zstdcli,
github, level 7 with dict, zstdcli, 40766 github, level 7 with dict, zstdcli, 40766
github, level 9, zstdcli, 137108 github, level 9, zstdcli, 137108
github, level 9 with dict, zstdcli, 41326 github, level 9 with dict, zstdcli, 41326
github, level 13, zstdcli, 135741 github, level 13, zstdcli, 135717
github, level 13 with dict, zstdcli, 41670 github, level 13 with dict, zstdcli, 41716
github, level 16, zstdcli, 135741 github, level 16, zstdcli, 136846
github, level 16 with dict, zstdcli, 39940 github, level 16 with dict, zstdcli, 39577
github, level 19, zstdcli, 135717 github, level 19, zstdcli, 136676
github, level 19 with dict, zstdcli, 39576 github, level 19 with dict, zstdcli, 39576
silesia, level -5, advanced one pass, 7152294 silesia, level -5, advanced one pass, 7152294
silesia, level -3, advanced one pass, 6789969 silesia, level -3, advanced one pass, 6789969
@ -119,29 +119,29 @@ silesia, level 0, advanced one pass,
silesia, level 1, advanced one pass, 5318036 silesia, level 1, advanced one pass, 5318036
silesia, level 3, advanced one pass, 4862377 silesia, level 3, advanced one pass, 4862377
silesia, level 4, advanced one pass, 4800629 silesia, level 4, advanced one pass, 4800629
silesia, level 5, advanced one pass, 4715005 silesia, level 5, advanced one pass, 4710178
silesia, level 6, advanced one pass, 4644055 silesia, level 6, advanced one pass, 4659996
silesia, level 7, advanced one pass, 4581559 silesia, level 7, advanced one pass, 4596234
silesia, level 9, advanced one pass, 4543862 silesia, level 9, advanced one pass, 4543862
silesia, level 13, advanced one pass, 4493931 silesia, level 13, advanced one pass, 4482073
silesia, level 16, advanced one pass, 4381885 silesia, level 16, advanced one pass, 4377391
silesia, level 19, advanced one pass, 4296899 silesia, level 19, advanced one pass, 4294841
silesia, no source size, advanced one pass, 4862377 silesia, no source size, advanced one pass, 4862377
silesia.tar, level -5, advanced one pass, 7160438 silesia.tar, level -5, advanced one pass, 7160438
silesia.tar, level -3, advanced one pass, 6789024 silesia.tar, level -3, advanced one pass, 6789024
silesia.tar, level -1, advanced one pass, 6195462 silesia.tar, level -1, advanced one pass, 6195462
silesia.tar, level 0, advanced one pass, 4875071 silesia.tar, level 0, advanced one pass, 4875008
silesia.tar, level 1, advanced one pass, 5339697 silesia.tar, level 1, advanced one pass, 5339697
silesia.tar, level 3, advanced one pass, 4875071 silesia.tar, level 3, advanced one pass, 4875008
silesia.tar, level 4, advanced one pass, 4813104 silesia.tar, level 4, advanced one pass, 4813507
silesia.tar, level 5, advanced one pass, 4726961 silesia.tar, level 5, advanced one pass, 4722235
silesia.tar, level 6, advanced one pass, 4654401 silesia.tar, level 6, advanced one pass, 4672194
silesia.tar, level 7, advanced one pass, 4591933 silesia.tar, level 7, advanced one pass, 4606658
silesia.tar, level 9, advanced one pass, 4554098 silesia.tar, level 9, advanced one pass, 4554098
silesia.tar, level 13, advanced one pass, 4503496 silesia.tar, level 13, advanced one pass, 4491702
silesia.tar, level 16, advanced one pass, 4387233 silesia.tar, level 16, advanced one pass, 4381277
silesia.tar, level 19, advanced one pass, 4283123 silesia.tar, level 19, advanced one pass, 4281581
silesia.tar, no source size, advanced one pass, 4875071 silesia.tar, no source size, advanced one pass, 4875008
github, level -5, advanced one pass, 232744 github, level -5, advanced one pass, 232744
github, level -5 with dict, advanced one pass, 45528 github, level -5 with dict, advanced one pass, 45528
github, level -3, advanced one pass, 220611 github, level -3, advanced one pass, 220611
@ -164,11 +164,11 @@ github, level 7, advanced one pass,
github, level 7 with dict, advanced one pass, 38766 github, level 7 with dict, advanced one pass, 38766
github, level 9, advanced one pass, 135108 github, level 9, advanced one pass, 135108
github, level 9 with dict, advanced one pass, 39326 github, level 9 with dict, advanced one pass, 39326
github, level 13, advanced one pass, 133741 github, level 13, advanced one pass, 133717
github, level 13 with dict, advanced one pass, 39670 github, level 13 with dict, advanced one pass, 39716
github, level 16, advanced one pass, 133741 github, level 16, advanced one pass, 134844
github, level 16 with dict, advanced one pass, 37940 github, level 16 with dict, advanced one pass, 37577
github, level 19, advanced one pass, 133717 github, level 19, advanced one pass, 134675
github, level 19 with dict, advanced one pass, 37576 github, level 19 with dict, advanced one pass, 37576
github, no source size, advanced one pass, 136397 github, no source size, advanced one pass, 136397
silesia, level -5, advanced one pass small out, 7152294 silesia, level -5, advanced one pass small out, 7152294
@ -178,29 +178,29 @@ silesia, level 0, advanced one pass smal
silesia, level 1, advanced one pass small out, 5318036 silesia, level 1, advanced one pass small out, 5318036
silesia, level 3, advanced one pass small out, 4862377 silesia, level 3, advanced one pass small out, 4862377
silesia, level 4, advanced one pass small out, 4800629 silesia, level 4, advanced one pass small out, 4800629
silesia, level 5, advanced one pass small out, 4715005 silesia, level 5, advanced one pass small out, 4710178
silesia, level 6, advanced one pass small out, 4644055 silesia, level 6, advanced one pass small out, 4659996
silesia, level 7, advanced one pass small out, 4581559 silesia, level 7, advanced one pass small out, 4596234
silesia, level 9, advanced one pass small out, 4543862 silesia, level 9, advanced one pass small out, 4543862
silesia, level 13, advanced one pass small out, 4493931 silesia, level 13, advanced one pass small out, 4482073
silesia, level 16, advanced one pass small out, 4381885 silesia, level 16, advanced one pass small out, 4377391
silesia, level 19, advanced one pass small out, 4296899 silesia, level 19, advanced one pass small out, 4294841
silesia, no source size, advanced one pass small out, 4862377 silesia, no source size, advanced one pass small out, 4862377
silesia.tar, level -5, advanced one pass small out, 7160438 silesia.tar, level -5, advanced one pass small out, 7160438
silesia.tar, level -3, advanced one pass small out, 6789024 silesia.tar, level -3, advanced one pass small out, 6789024
silesia.tar, level -1, advanced one pass small out, 6195462 silesia.tar, level -1, advanced one pass small out, 6195462
silesia.tar, level 0, advanced one pass small out, 4875071 silesia.tar, level 0, advanced one pass small out, 4875008
silesia.tar, level 1, advanced one pass small out, 5339697 silesia.tar, level 1, advanced one pass small out, 5339697
silesia.tar, level 3, advanced one pass small out, 4875071 silesia.tar, level 3, advanced one pass small out, 4875008
silesia.tar, level 4, advanced one pass small out, 4813104 silesia.tar, level 4, advanced one pass small out, 4813507
silesia.tar, level 5, advanced one pass small out, 4726961 silesia.tar, level 5, advanced one pass small out, 4722235
silesia.tar, level 6, advanced one pass small out, 4654401 silesia.tar, level 6, advanced one pass small out, 4672194
silesia.tar, level 7, advanced one pass small out, 4591933 silesia.tar, level 7, advanced one pass small out, 4606658
silesia.tar, level 9, advanced one pass small out, 4554098 silesia.tar, level 9, advanced one pass small out, 4554098
silesia.tar, level 13, advanced one pass small out, 4503496 silesia.tar, level 13, advanced one pass small out, 4491702
silesia.tar, level 16, advanced one pass small out, 4387233 silesia.tar, level 16, advanced one pass small out, 4381277
silesia.tar, level 19, advanced one pass small out, 4283123 silesia.tar, level 19, advanced one pass small out, 4281581
silesia.tar, no source size, advanced one pass small out, 4875071 silesia.tar, no source size, advanced one pass small out, 4875008
github, level -5, advanced one pass small out, 232744 github, level -5, advanced one pass small out, 232744
github, level -5 with dict, advanced one pass small out, 45528 github, level -5 with dict, advanced one pass small out, 45528
github, level -3, advanced one pass small out, 220611 github, level -3, advanced one pass small out, 220611
@ -223,11 +223,11 @@ github, level 7, advanced one pass smal
github, level 7 with dict, advanced one pass small out, 38766 github, level 7 with dict, advanced one pass small out, 38766
github, level 9, advanced one pass small out, 135108 github, level 9, advanced one pass small out, 135108
github, level 9 with dict, advanced one pass small out, 39326 github, level 9 with dict, advanced one pass small out, 39326
github, level 13, advanced one pass small out, 133741 github, level 13, advanced one pass small out, 133717
github, level 13 with dict, advanced one pass small out, 39670 github, level 13 with dict, advanced one pass small out, 39716
github, level 16, advanced one pass small out, 133741 github, level 16, advanced one pass small out, 134844
github, level 16 with dict, advanced one pass small out, 37940 github, level 16 with dict, advanced one pass small out, 37577
github, level 19, advanced one pass small out, 133717 github, level 19, advanced one pass small out, 134675
github, level 19 with dict, advanced one pass small out, 37576 github, level 19 with dict, advanced one pass small out, 37576
github, no source size, advanced one pass small out, 136397 github, no source size, advanced one pass small out, 136397
silesia, level -5, advanced streaming, 7152294 silesia, level -5, advanced streaming, 7152294
@ -237,29 +237,29 @@ silesia, level 0, advanced streaming,
silesia, level 1, advanced streaming, 5318036 silesia, level 1, advanced streaming, 5318036
silesia, level 3, advanced streaming, 4862377 silesia, level 3, advanced streaming, 4862377
silesia, level 4, advanced streaming, 4800629 silesia, level 4, advanced streaming, 4800629
silesia, level 5, advanced streaming, 4715005 silesia, level 5, advanced streaming, 4710178
silesia, level 6, advanced streaming, 4644055 silesia, level 6, advanced streaming, 4659996
silesia, level 7, advanced streaming, 4581559 silesia, level 7, advanced streaming, 4596234
silesia, level 9, advanced streaming, 4543862 silesia, level 9, advanced streaming, 4543862
silesia, level 13, advanced streaming, 4493931 silesia, level 13, advanced streaming, 4482073
silesia, level 16, advanced streaming, 4381885 silesia, level 16, advanced streaming, 4377391
silesia, level 19, advanced streaming, 4296899 silesia, level 19, advanced streaming, 4294841
silesia, no source size, advanced streaming, 4862341 silesia, no source size, advanced streaming, 4862341
silesia.tar, level -5, advanced streaming, 7160440 silesia.tar, level -5, advanced streaming, 7160440
silesia.tar, level -3, advanced streaming, 6789026 silesia.tar, level -3, advanced streaming, 6789026
silesia.tar, level -1, advanced streaming, 6195465 silesia.tar, level -1, advanced streaming, 6195465
silesia.tar, level 0, advanced streaming, 4875071 silesia.tar, level 0, advanced streaming, 4875010
silesia.tar, level 1, advanced streaming, 5339701 silesia.tar, level 1, advanced streaming, 5339701
silesia.tar, level 3, advanced streaming, 4875071 silesia.tar, level 3, advanced streaming, 4875010
silesia.tar, level 4, advanced streaming, 4813104 silesia.tar, level 4, advanced streaming, 4813507
silesia.tar, level 5, advanced streaming, 4726977 silesia.tar, level 5, advanced streaming, 4722240
silesia.tar, level 6, advanced streaming, 4654404 silesia.tar, level 6, advanced streaming, 4672203
silesia.tar, level 7, advanced streaming, 4591934 silesia.tar, level 7, advanced streaming, 4606658
silesia.tar, level 9, advanced streaming, 4554105 silesia.tar, level 9, advanced streaming, 4554105
silesia.tar, level 13, advanced streaming, 4503496 silesia.tar, level 13, advanced streaming, 4491703
silesia.tar, level 16, advanced streaming, 4387233 silesia.tar, level 16, advanced streaming, 4381277
silesia.tar, level 19, advanced streaming, 4283123 silesia.tar, level 19, advanced streaming, 4281581
silesia.tar, no source size, advanced streaming, 4875067 silesia.tar, no source size, advanced streaming, 4875006
github, level -5, advanced streaming, 232744 github, level -5, advanced streaming, 232744
github, level -5 with dict, advanced streaming, 45528 github, level -5 with dict, advanced streaming, 45528
github, level -3, advanced streaming, 220611 github, level -3, advanced streaming, 220611
@ -282,11 +282,11 @@ github, level 7, advanced streaming,
github, level 7 with dict, advanced streaming, 38766 github, level 7 with dict, advanced streaming, 38766
github, level 9, advanced streaming, 135108 github, level 9, advanced streaming, 135108
github, level 9 with dict, advanced streaming, 39326 github, level 9 with dict, advanced streaming, 39326
github, level 13, advanced streaming, 133741 github, level 13, advanced streaming, 133717
github, level 13 with dict, advanced streaming, 39670 github, level 13 with dict, advanced streaming, 39716
github, level 16, advanced streaming, 133741 github, level 16, advanced streaming, 134844
github, level 16 with dict, advanced streaming, 37940 github, level 16 with dict, advanced streaming, 37577
github, level 19, advanced streaming, 133717 github, level 19, advanced streaming, 134675
github, level 19 with dict, advanced streaming, 37576 github, level 19 with dict, advanced streaming, 37576
github, no source size, advanced streaming, 136397 github, no source size, advanced streaming, 136397
silesia, level -5, old streaming, 7152294 silesia, level -5, old streaming, 7152294
@ -296,29 +296,29 @@ silesia, level 0, old streaming,
silesia, level 1, old streaming, 5318036 silesia, level 1, old streaming, 5318036
silesia, level 3, old streaming, 4862377 silesia, level 3, old streaming, 4862377
silesia, level 4, old streaming, 4800629 silesia, level 4, old streaming, 4800629
silesia, level 5, old streaming, 4715005 silesia, level 5, old streaming, 4710178
silesia, level 6, old streaming, 4644055 silesia, level 6, old streaming, 4659996
silesia, level 7, old streaming, 4581559 silesia, level 7, old streaming, 4596234
silesia, level 9, old streaming, 4543862 silesia, level 9, old streaming, 4543862
silesia, level 13, old streaming, 4493931 silesia, level 13, old streaming, 4482073
silesia, level 16, old streaming, 4381885 silesia, level 16, old streaming, 4377391
silesia, level 19, old streaming, 4296899 silesia, level 19, old streaming, 4294841
silesia, no source size, old streaming, 4862341 silesia, no source size, old streaming, 4862341
silesia.tar, level -5, old streaming, 7160440 silesia.tar, level -5, old streaming, 7160440
silesia.tar, level -3, old streaming, 6789026 silesia.tar, level -3, old streaming, 6789026
silesia.tar, level -1, old streaming, 6195465 silesia.tar, level -1, old streaming, 6195465
silesia.tar, level 0, old streaming, 4875071 silesia.tar, level 0, old streaming, 4875010
silesia.tar, level 1, old streaming, 5339701 silesia.tar, level 1, old streaming, 5339701
silesia.tar, level 3, old streaming, 4875071 silesia.tar, level 3, old streaming, 4875010
silesia.tar, level 4, old streaming, 4813104 silesia.tar, level 4, old streaming, 4813507
silesia.tar, level 5, old streaming, 4726977 silesia.tar, level 5, old streaming, 4722240
silesia.tar, level 6, old streaming, 4654404 silesia.tar, level 6, old streaming, 4672203
silesia.tar, level 7, old streaming, 4591934 silesia.tar, level 7, old streaming, 4606658
silesia.tar, level 9, old streaming, 4554105 silesia.tar, level 9, old streaming, 4554105
silesia.tar, level 13, old streaming, 4503496 silesia.tar, level 13, old streaming, 4491703
silesia.tar, level 16, old streaming, 4387233 silesia.tar, level 16, old streaming, 4381277
silesia.tar, level 19, old streaming, 4283123 silesia.tar, level 19, old streaming, 4281581
silesia.tar, no source size, old streaming, 4875067 silesia.tar, no source size, old streaming, 4875006
github, level -5, old streaming, 232744 github, level -5, old streaming, 232744
github, level -5 with dict, old streaming, 45528 github, level -5 with dict, old streaming, 45528
github, level -3, old streaming, 220611 github, level -3, old streaming, 220611
@ -341,10 +341,10 @@ github, level 7, old streaming,
github, level 7 with dict, old streaming, 38766 github, level 7 with dict, old streaming, 38766
github, level 9, old streaming, 135108 github, level 9, old streaming, 135108
github, level 9 with dict, old streaming, 39326 github, level 9 with dict, old streaming, 39326
github, level 13, old streaming, 133741 github, level 13, old streaming, 133717
github, level 13 with dict, old streaming, 39670 github, level 13 with dict, old streaming, 39716
github, level 16, old streaming, 133741 github, level 16, old streaming, 134846
github, level 16 with dict, old streaming, 37940 github, level 16 with dict, old streaming, 37577
github, level 19, old streaming, 133717 github, level 19, old streaming, 134676
github, level 19 with dict, old streaming, 37576 github, level 19 with dict, old streaming, 37576
github, no source size, old streaming, 141003 github, no source size, old streaming, 141003

1 Data Config Method Total compressed size
2 silesia.tar level -5 compress simple 7160438
3 silesia.tar level -3 compress simple 6789024
4 silesia.tar level -1 compress simple 6195462
5 silesia.tar level 0 compress simple 4875071 4875008
6 silesia.tar level 1 compress simple 5339697
7 silesia.tar level 3 compress simple 4875071 4875008
8 silesia.tar level 4 compress simple 4813104 4813507
9 silesia.tar level 5 compress simple 4726961 4722235
10 silesia.tar level 6 compress simple 4654401 4672194
11 silesia.tar level 7 compress simple 4591933 4606658
12 silesia.tar level 9 compress simple 4554098
13 silesia.tar level 13 compress simple 4503496 4491702
14 silesia.tar level 16 compress simple 4387233 4381277
15 silesia.tar level 19 compress simple 4283123 4281581
16 silesia level -5 compress cctx 7152294
17 silesia level -3 compress cctx 6789969
18 silesia level -1 compress cctx 6191548
20 silesia level 1 compress cctx 5318036
21 silesia level 3 compress cctx 4862377
22 silesia level 4 compress cctx 4800629
23 silesia level 5 compress cctx 4715005 4710178
24 silesia level 6 compress cctx 4644055 4659996
25 silesia level 7 compress cctx 4581559 4596234
26 silesia level 9 compress cctx 4543862
27 silesia level 13 compress cctx 4493931 4482073
28 silesia level 16 compress cctx 4381885 4377391
29 silesia level 19 compress cctx 4296899 4294841
30 github level -5 compress cctx 232744
31 github level -5 with dict compress cctx 45704
32 github level -3 compress cctx 220611
49 github level 7 with dict compress cctx 38741
50 github level 9 compress cctx 135108
51 github level 9 with dict compress cctx 39335
52 github level 13 compress cctx 133741 133717
53 github level 13 with dict compress cctx 39670 39923
54 github level 16 compress cctx 133741 134844
55 github level 16 with dict compress cctx 37928 37568
56 github level 19 compress cctx 133717 134675
57 github level 19 with dict compress cctx 37567
58 silesia level -5 zstdcli 7152342
59 silesia level -3 zstdcli 6790021
62 silesia level 1 zstdcli 5318084
63 silesia level 3 zstdcli 4862425
64 silesia level 4 zstdcli 4800677
65 silesia level 5 zstdcli 4715053 4710226
66 silesia level 6 zstdcli 4644103 4660044
67 silesia level 7 zstdcli 4581607 4596282
68 silesia level 9 zstdcli 4543910
69 silesia level 13 zstdcli 4493979 4482121
70 silesia level 16 zstdcli 4381933 4377439
71 silesia level 19 zstdcli 4296947 4294889
72 silesia.tar level -5 zstdcli 7159586
73 silesia.tar level -3 zstdcli 6791018
74 silesia.tar level -1 zstdcli 6196283
75 silesia.tar level 0 zstdcli 4876730 4875213
76 silesia.tar level 1 zstdcli 5340312
77 silesia.tar level 3 zstdcli 4876730 4875213
78 silesia.tar level 4 zstdcli 4817723 4814545
79 silesia.tar level 5 zstdcli 4730389 4723284
80 silesia.tar level 6 zstdcli 4655708 4673591
81 silesia.tar level 7 zstdcli 4593407 4608342
82 silesia.tar level 9 zstdcli 4556135
83 silesia.tar level 13 zstdcli 4503500 4491706
84 silesia.tar level 16 zstdcli 4387237 4381281
85 silesia.tar level 19 zstdcli 4283127 4281585
86 silesia.tar no source size zstdcli 4876726 4875209
87 github level -5 zstdcli 234744
88 github level -5 with dict zstdcli 47528
89 github level -3 zstdcli 222611
106 github level 7 with dict zstdcli 40766
107 github level 9 zstdcli 137108
108 github level 9 with dict zstdcli 41326
109 github level 13 zstdcli 135741 135717
110 github level 13 with dict zstdcli 41670 41716
111 github level 16 zstdcli 135741 136846
112 github level 16 with dict zstdcli 39940 39577
113 github level 19 zstdcli 135717 136676
114 github level 19 with dict zstdcli 39576
115 silesia level -5 advanced one pass 7152294
116 silesia level -3 advanced one pass 6789969
119 silesia level 1 advanced one pass 5318036
120 silesia level 3 advanced one pass 4862377
121 silesia level 4 advanced one pass 4800629
122 silesia level 5 advanced one pass 4715005 4710178
123 silesia level 6 advanced one pass 4644055 4659996
124 silesia level 7 advanced one pass 4581559 4596234
125 silesia level 9 advanced one pass 4543862
126 silesia level 13 advanced one pass 4493931 4482073
127 silesia level 16 advanced one pass 4381885 4377391
128 silesia level 19 advanced one pass 4296899 4294841
129 silesia no source size advanced one pass 4862377
130 silesia.tar level -5 advanced one pass 7160438
131 silesia.tar level -3 advanced one pass 6789024
132 silesia.tar level -1 advanced one pass 6195462
133 silesia.tar level 0 advanced one pass 4875071 4875008
134 silesia.tar level 1 advanced one pass 5339697
135 silesia.tar level 3 advanced one pass 4875071 4875008
136 silesia.tar level 4 advanced one pass 4813104 4813507
137 silesia.tar level 5 advanced one pass 4726961 4722235
138 silesia.tar level 6 advanced one pass 4654401 4672194
139 silesia.tar level 7 advanced one pass 4591933 4606658
140 silesia.tar level 9 advanced one pass 4554098
141 silesia.tar level 13 advanced one pass 4503496 4491702
142 silesia.tar level 16 advanced one pass 4387233 4381277
143 silesia.tar level 19 advanced one pass 4283123 4281581
144 silesia.tar no source size advanced one pass 4875071 4875008
145 github level -5 advanced one pass 232744
146 github level -5 with dict advanced one pass 45528
147 github level -3 advanced one pass 220611
164 github level 7 with dict advanced one pass 38766
165 github level 9 advanced one pass 135108
166 github level 9 with dict advanced one pass 39326
167 github level 13 advanced one pass 133741 133717
168 github level 13 with dict advanced one pass 39670 39716
169 github level 16 advanced one pass 133741 134844
170 github level 16 with dict advanced one pass 37940 37577
171 github level 19 advanced one pass 133717 134675
172 github level 19 with dict advanced one pass 37576
173 github no source size advanced one pass 136397
174 silesia level -5 advanced one pass small out 7152294
178 silesia level 1 advanced one pass small out 5318036
179 silesia level 3 advanced one pass small out 4862377
180 silesia level 4 advanced one pass small out 4800629
181 silesia level 5 advanced one pass small out 4715005 4710178
182 silesia level 6 advanced one pass small out 4644055 4659996
183 silesia level 7 advanced one pass small out 4581559 4596234
184 silesia level 9 advanced one pass small out 4543862
185 silesia level 13 advanced one pass small out 4493931 4482073
186 silesia level 16 advanced one pass small out 4381885 4377391
187 silesia level 19 advanced one pass small out 4296899 4294841
188 silesia no source size advanced one pass small out 4862377
189 silesia.tar level -5 advanced one pass small out 7160438
190 silesia.tar level -3 advanced one pass small out 6789024
191 silesia.tar level -1 advanced one pass small out 6195462
192 silesia.tar level 0 advanced one pass small out 4875071 4875008
193 silesia.tar level 1 advanced one pass small out 5339697
194 silesia.tar level 3 advanced one pass small out 4875071 4875008
195 silesia.tar level 4 advanced one pass small out 4813104 4813507
196 silesia.tar level 5 advanced one pass small out 4726961 4722235
197 silesia.tar level 6 advanced one pass small out 4654401 4672194
198 silesia.tar level 7 advanced one pass small out 4591933 4606658
199 silesia.tar level 9 advanced one pass small out 4554098
200 silesia.tar level 13 advanced one pass small out 4503496 4491702
201 silesia.tar level 16 advanced one pass small out 4387233 4381277
202 silesia.tar level 19 advanced one pass small out 4283123 4281581
203 silesia.tar no source size advanced one pass small out 4875071 4875008
204 github level -5 advanced one pass small out 232744
205 github level -5 with dict advanced one pass small out 45528
206 github level -3 advanced one pass small out 220611
223 github level 7 with dict advanced one pass small out 38766
224 github level 9 advanced one pass small out 135108
225 github level 9 with dict advanced one pass small out 39326
226 github level 13 advanced one pass small out 133741 133717
227 github level 13 with dict advanced one pass small out 39670 39716
228 github level 16 advanced one pass small out 133741 134844
229 github level 16 with dict advanced one pass small out 37940 37577
230 github level 19 advanced one pass small out 133717 134675
231 github level 19 with dict advanced one pass small out 37576
232 github no source size advanced one pass small out 136397
233 silesia level -5 advanced streaming 7152294
237 silesia level 1 advanced streaming 5318036
238 silesia level 3 advanced streaming 4862377
239 silesia level 4 advanced streaming 4800629
240 silesia level 5 advanced streaming 4715005 4710178
241 silesia level 6 advanced streaming 4644055 4659996
242 silesia level 7 advanced streaming 4581559 4596234
243 silesia level 9 advanced streaming 4543862
244 silesia level 13 advanced streaming 4493931 4482073
245 silesia level 16 advanced streaming 4381885 4377391
246 silesia level 19 advanced streaming 4296899 4294841
247 silesia no source size advanced streaming 4862341
248 silesia.tar level -5 advanced streaming 7160440
249 silesia.tar level -3 advanced streaming 6789026
250 silesia.tar level -1 advanced streaming 6195465
251 silesia.tar level 0 advanced streaming 4875071 4875010
252 silesia.tar level 1 advanced streaming 5339701
253 silesia.tar level 3 advanced streaming 4875071 4875010
254 silesia.tar level 4 advanced streaming 4813104 4813507
255 silesia.tar level 5 advanced streaming 4726977 4722240
256 silesia.tar level 6 advanced streaming 4654404 4672203
257 silesia.tar level 7 advanced streaming 4591934 4606658
258 silesia.tar level 9 advanced streaming 4554105
259 silesia.tar level 13 advanced streaming 4503496 4491703
260 silesia.tar level 16 advanced streaming 4387233 4381277
261 silesia.tar level 19 advanced streaming 4283123 4281581
262 silesia.tar no source size advanced streaming 4875067 4875006
263 github level -5 advanced streaming 232744
264 github level -5 with dict advanced streaming 45528
265 github level -3 advanced streaming 220611
282 github level 7 with dict advanced streaming 38766
283 github level 9 advanced streaming 135108
284 github level 9 with dict advanced streaming 39326
285 github level 13 advanced streaming 133741 133717
286 github level 13 with dict advanced streaming 39670 39716
287 github level 16 advanced streaming 133741 134844
288 github level 16 with dict advanced streaming 37940 37577
289 github level 19 advanced streaming 133717 134675
290 github level 19 with dict advanced streaming 37576
291 github no source size advanced streaming 136397
292 silesia level -5 old streaming 7152294
296 silesia level 1 old streaming 5318036
297 silesia level 3 old streaming 4862377
298 silesia level 4 old streaming 4800629
299 silesia level 5 old streaming 4715005 4710178
300 silesia level 6 old streaming 4644055 4659996
301 silesia level 7 old streaming 4581559 4596234
302 silesia level 9 old streaming 4543862
303 silesia level 13 old streaming 4493931 4482073
304 silesia level 16 old streaming 4381885 4377391
305 silesia level 19 old streaming 4296899 4294841
306 silesia no source size old streaming 4862341
307 silesia.tar level -5 old streaming 7160440
308 silesia.tar level -3 old streaming 6789026
309 silesia.tar level -1 old streaming 6195465
310 silesia.tar level 0 old streaming 4875071 4875010
311 silesia.tar level 1 old streaming 5339701
312 silesia.tar level 3 old streaming 4875071 4875010
313 silesia.tar level 4 old streaming 4813104 4813507
314 silesia.tar level 5 old streaming 4726977 4722240
315 silesia.tar level 6 old streaming 4654404 4672203
316 silesia.tar level 7 old streaming 4591934 4606658
317 silesia.tar level 9 old streaming 4554105
318 silesia.tar level 13 old streaming 4503496 4491703
319 silesia.tar level 16 old streaming 4387233 4381277
320 silesia.tar level 19 old streaming 4283123 4281581
321 silesia.tar no source size old streaming 4875067 4875006
322 github level -5 old streaming 232744
323 github level -5 with dict old streaming 45528
324 github level -3 old streaming 220611
341 github level 7 with dict old streaming 38766
342 github level 9 old streaming 135108
343 github level 9 with dict old streaming 39326
344 github level 13 old streaming 133741 133717
345 github level 13 with dict old streaming 39670 39716
346 github level 16 old streaming 133741 134846
347 github level 16 with dict old streaming 37940 37577
348 github level 19 old streaming 133717 134676
349 github level 19 with dict old streaming 37576
350 github no source size old streaming 141003

4
tests/zstreamtest.c
View File

@ -230,7 +230,7 @@ static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog)); CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch)); CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength)); CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionStrategy, &value)); CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
savedParams->cParams.strategy = value; savedParams->cParams.strategy = value;
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag)); CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
@ -1548,7 +1548,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest,
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */ CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} } } } } }