facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1171 from facebook/negLevels

Browse Source 
Negative compression levels
This commit is contained in:
Yann Collet 2018-06-08 12:30:42 -07:00 committed by GitHub
commit c92ef71d86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 107 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/zstd_manual.html
View File

@ -867,12 +867,6 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
</b>/* experimental parameters - no stability guaranteed */<b> </b>/* experimental parameters - no stability guaranteed */<b>
</b>/* =================================================================== */<b> </b>/* =================================================================== */<b>
ZSTD_p_compressLiterals=1000, </b>/* control huffman compression of literals (enabled) by default.<b>
* disabling it improves speed and decreases compression ratio by a large amount.
* note : this setting is automatically updated when changing compression level.
* positive compression levels set ZSTD_p_compressLiterals to 1.
* negative compression levels set ZSTD_p_compressLiterals to 0. */
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b> ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b>
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */

109
lib/compress/zstd_compress.c
View File

@ -64,19 +64,26 @@ ZSTD_CCtx* ZSTD_createCCtx(void)
return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
} }
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager)
{ {
ZSTD_STATIC_ASSERT(zcss_init==0); assert(cctx != NULL);
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); memset(cctx, 0, sizeof(*cctx));
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; cctx->customMem = memManager;
{ ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_calloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL;
cctx->customMem = customMem;
cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
{ size_t const err = ZSTD_CCtx_resetParameters(cctx); { size_t const err = ZSTD_CCtx_resetParameters(cctx);
assert(!ZSTD_isError(err)); assert(!ZSTD_isError(err));
(void)err; (void)err;
} }
}
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_STATIC_ASSERT(zcss_init==0);
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL;
ZSTD_initCCtx(cctx, customMem);
return cctx; return cctx;
} }
} }
@ -104,17 +111,24 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
return cctx; return cctx;
} }
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
{ {
if (cctx==NULL) return 0; /* support free on NULL */ assert(cctx != NULL);
if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */ assert(cctx->staticSize == 0);
ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL;
ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL; ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL;
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
#endif #endif
}
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{
if (cctx==NULL) return 0; /* support free on NULL */
if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */
ZSTD_freeCCtxContent(cctx);
ZSTD_free(cctx, cctx->customMem); ZSTD_free(cctx, cctx->customMem);
return 0; /* reserved as a potential error code in the future */ return 0;
} }
@ -253,7 +267,6 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_p_minMatch: case ZSTD_p_minMatch:
case ZSTD_p_targetLength: case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy: case ZSTD_p_compressionStrategy:
case ZSTD_p_compressLiterals:
return 1; return 1;
case ZSTD_p_format: case ZSTD_p_format:
@ -304,7 +317,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_compressLiterals:
case ZSTD_p_contentSizeFlag: case ZSTD_p_contentSizeFlag:
case ZSTD_p_checksumFlag: case ZSTD_p_checksumFlag:
case ZSTD_p_dictIDFlag: case ZSTD_p_dictIDFlag:
@ -353,7 +365,6 @@ size_t ZSTD_CCtxParam_setParameter(
int cLevel = (int)value; /* cast expected to restore negative sign */ int cLevel = (int)value; /* cast expected to restore negative sign */
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevel) { /* 0 : does not change current level */ if (cLevel) { /* 0 : does not change current level */
CCtxParams->disableLiteralCompression = (cLevel<0); /* negative levels disable huffman */
CCtxParams->compressionLevel = cLevel; CCtxParams->compressionLevel = cLevel;
} }
if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel; if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel;
@ -401,10 +412,6 @@ size_t ZSTD_CCtxParam_setParameter(
CCtxParams->cParams.strategy = (ZSTD_strategy)value; CCtxParams->cParams.strategy = (ZSTD_strategy)value;
return (size_t)CCtxParams->cParams.strategy; return (size_t)CCtxParams->cParams.strategy;
case ZSTD_p_compressLiterals:
CCtxParams->disableLiteralCompression = !value;
return !CCtxParams->disableLiteralCompression;
case ZSTD_p_contentSizeFlag : case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */ /* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(4, "set content size flag = %u", (value>0)); DEBUGLOG(4, "set content size flag = %u", (value>0));
@ -516,9 +523,6 @@ size_t ZSTD_CCtxParam_getParameter(
case ZSTD_p_compressionStrategy : case ZSTD_p_compressionStrategy :
*value = (unsigned)CCtxParams->cParams.strategy; *value = (unsigned)CCtxParams->cParams.strategy;
break; break;
case ZSTD_p_compressLiterals:
*value = !CCtxParams->disableLiteralCompression;
break;
case ZSTD_p_contentSizeFlag : case ZSTD_p_contentSizeFlag :
*value = CCtxParams->fParams.contentSizeFlag; *value = CCtxParams->fParams.contentSizeFlag;
break; break;
@ -685,8 +689,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
if ((U32)(cParams.targetLength) < ZSTD_TARGETLENGTH_MIN)
return ERROR(parameter_unsupported);
if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) if ((U32)(cParams.strategy) > (U32)ZSTD_btultra)
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
return 0; return 0;
@ -707,8 +709,6 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
if ((U32)(cParams.targetLength) < ZSTD_TARGETLENGTH_MIN)
cParams.targetLength = ZSTD_TARGETLENGTH_MIN;
CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra); CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
return cParams; return cParams;
} }
@ -2084,9 +2084,10 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
/* Compress literals */ /* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart; { const BYTE* const literals = seqStorePtr->litStart;
size_t const litSize = seqStorePtr->lit - literals; size_t const litSize = seqStorePtr->lit - literals;
int const disableLiteralCompression = (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
size_t const cSize = ZSTD_compressLiterals( size_t const cSize = ZSTD_compressLiterals(
&prevEntropy->huf, &nextEntropy->huf, &prevEntropy->huf, &nextEntropy->huf,
cctxParams->cParams.strategy, cctxParams->disableLiteralCompression, cctxParams->cParams.strategy, disableLiteralCompression,
op, dstCapacity, op, dstCapacity,
literals, litSize, literals, litSize,
workspace, bmi2); workspace, bmi2);
@ -2947,7 +2948,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
cctxParams); cctxParams);
} }
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
@ -2955,7 +2956,11 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
{ {
DEBUGLOG(4, "ZSTD_compress_advanced"); DEBUGLOG(4, "ZSTD_compress_advanced");
CHECK_F(ZSTD_checkCParams(params.cParams)); CHECK_F(ZSTD_checkCParams(params.cParams));
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(cctx,
dst, dstCapacity,
src, srcSize,
dict, dictSize,
params);
} }
/* Internal */ /* Internal */
@ -2966,37 +2971,44 @@ size_t ZSTD_compress_advanced_internal(
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_CCtx_params params) ZSTD_CCtx_params params)
{ {
DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (U32)srcSize);
(U32)srcSize); CHECK_F( ZSTD_compressBegin_internal(cctx,
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
params, srcSize, ZSTDb_not_buffered) ); params, srcSize, ZSTDb_not_buffered) );
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, int compressionLevel) void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel)
{ {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize ? srcSize : 1, dict ? dictSize : 0); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0);
ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
assert(params.fParams.contentSizeFlag == 1); assert(params.fParams.contentSizeFlag == 1);
ZSTD_CCtxParam_setParameter(&cctxParams, ZSTD_p_compressLiterals, compressionLevel>=0);
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams);
} }
size_t ZSTD_compressCCtx (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel)
{ {
DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (U32)srcSize); DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (U32)srcSize);
assert(cctx != NULL);
return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
} }
size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) size_t ZSTD_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel)
{ {
size_t result; size_t result;
ZSTD_CCtx ctxBody; ZSTD_CCtx ctxBody;
memset(&ctxBody, 0, sizeof(ctxBody)); ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem);
ctxBody.customMem = ZSTD_defaultCMem;
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
ZSTD_free(ctxBody.workSpace, ZSTD_defaultCMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */
return result; return result;
} }
@ -3297,8 +3309,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
const ZSTD_CDict* const cdict, const ZSTD_CDict* const cdict,
ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize) ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_resetCStream_internal (disableLiteralCompression=%i)", DEBUGLOG(4, "ZSTD_resetCStream_internal");
params.disableLiteralCompression);
/* params are supposed to be fully validated at this point */ /* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -3728,8 +3739,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ /* "default" - guarantees a monotonically increasing memory budget */ { /* "default" - guarantees a monotonically increasing memory budget */
/* W, C, H, S, L, TL, strat */ /* W, C, H, S, L, TL, strat */
{ 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
{ 19, 13, 14, 1, 7, 1, ZSTD_fast }, /* level 1 */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 1, ZSTD_fast }, /* level 2 */ { 19, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
{ 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ { 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */
{ 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ { 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */
{ 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */ { 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */
@ -3754,7 +3765,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ /* for srcSize <= 256 KB */ { /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 18, 13, 14, 1, 6, 1, ZSTD_fast }, /* level 1 */ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
{ 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */ { 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */
{ 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */ { 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */
{ 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/
@ -3780,8 +3791,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ /* for srcSize <= 128 KB */ { /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 17, 12, 13, 1, 6, 1, ZSTD_fast }, /* level 1 */ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
{ 17, 13, 15, 1, 5, 1, ZSTD_fast }, /* level 2 */ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
{ 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */ { 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */
{ 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */ { 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */
{ 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
@ -3806,8 +3817,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ /* for srcSize <= 16 KB */ { /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
{ 14, 14, 15, 1, 5, 1, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
{ 14, 14, 15, 1, 4, 1, ZSTD_fast }, /* level 2 */ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
{ 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/ { 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/
{ 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/
{ 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/

1
lib/compress/zstd_compress_internal.h
View File

@ -183,7 +183,6 @@ struct ZSTD_CCtx_params_s {
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
int compressionLevel; int compressionLevel;
int disableLiteralCompression;
int forceWindow; /* force back-references to respect limit of int forceWindow; /* force back-references to respect limit of
* 1<<wLog, even for dictionary */ * 1<<wLog, even for dictionary */

7
lib/compress/zstd_fast.c
View File

@ -45,7 +45,7 @@ FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_fast_generic( size_t ZSTD_compressBlock_fast_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, void const* src, size_t srcSize,
U32 const hlog, U32 const stepSize, U32 const mls, U32 const hlog, U32 stepSize, U32 const mls,
ZSTD_dictMode_e const dictMode) ZSTD_dictMode_e const dictMode)
{ {
U32* const hashTable = ms->hashTable; U32* const hashTable = ms->hashTable;
@ -84,6 +84,7 @@ size_t ZSTD_compressBlock_fast_generic(
|| prefixStartIndex >= (U32)(dictEnd - dictBase)); || prefixStartIndex >= (U32)(dictEnd - dictBase));
/* init */ /* init */
stepSize += !stepSize; /* support stepSize of 0 */
ip += (dictAndPrefixLength == 0); ip += (dictAndPrefixLength == 0);
if (dictMode == ZSTD_noDict) { if (dictMode == ZSTD_noDict) {
U32 const maxRep = (U32)(ip - prefixStart); U32 const maxRep = (U32)(ip - prefixStart);
@ -264,7 +265,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
static size_t ZSTD_compressBlock_fast_extDict_generic( static size_t ZSTD_compressBlock_fast_extDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, void const* src, size_t srcSize,
U32 const hlog, U32 const stepSize, U32 const mls) U32 const hlog, U32 stepSize, U32 const mls)
{ {
U32* hashTable = ms->hashTable; U32* hashTable = ms->hashTable;
const BYTE* const base = ms->window.base; const BYTE* const base = ms->window.base;
@ -281,6 +282,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const ilimit = iend - 8; const BYTE* const ilimit = iend - 8;
U32 offset_1=rep[0], offset_2=rep[1]; U32 offset_1=rep[0], offset_2=rep[1];
stepSize += !stepSize; /* support stepSize == 0 */
/* Search Loop */ /* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t h = ZSTD_hashPtr(ip, hlog, mls); const size_t h = ZSTD_hashPtr(ip, hlog, mls);

5
lib/compress/zstdmt_compress.c
View File

@ -961,7 +961,6 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params)
jobParams.cParams = params.cParams; jobParams.cParams = params.cParams;
jobParams.fParams = params.fParams; jobParams.fParams = params.fParams;
jobParams.compressionLevel = params.compressionLevel; jobParams.compressionLevel = params.compressionLevel;
jobParams.disableLiteralCompression = params.disableLiteralCompression;
return jobParams; return jobParams;
} }
@ -1222,8 +1221,8 @@ size_t ZSTDMT_initCStream_internal(
const ZSTD_CDict* cdict, ZSTD_CCtx_params params, const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u, disableLiteralCompression=%i)", DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)",
(U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx, params.disableLiteralCompression); (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx);
/* params are supposed to be fully validated at this point */ /* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */

7
lib/zstd.h
View File

@ -395,7 +395,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MIN 1 /* only used by btopt, btultra and btfast */
#define ZSTD_LDM_MINMATCH_MIN 4 #define ZSTD_LDM_MINMATCH_MIN 4
#define ZSTD_LDM_MINMATCH_MAX 4096 #define ZSTD_LDM_MINMATCH_MAX 4096
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
@ -1052,12 +1051,6 @@ typedef enum {
/* experimental parameters - no stability guaranteed */ /* experimental parameters - no stability guaranteed */
/* =================================================================== */ /* =================================================================== */
ZSTD_p_compressLiterals=1000, /* control huffman compression of literals (enabled) by default.
* disabling it improves speed and decreases compression ratio by a large amount.
* note : this setting is automatically updated when changing compression level.
* positive compression levels set ZSTD_p_compressLiterals to 1.
* negative compression levels set ZSTD_p_compressLiterals to 0. */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */

3
tests/fuzz/zstd_helpers.c
View File

@ -34,8 +34,7 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN, cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
ZSTD_SEARCHLENGTH_MAX); ZSTD_SEARCHLENGTH_MAX);
cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN, cParams.targetLength = FUZZ_rand32(state, 0, 512);
512);
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra); cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
return ZSTD_adjustCParams(cParams, srcSize, 0); return ZSTD_adjustCParams(cParams, srcSize, 0);
} }

30
tests/fuzzer.c
View File

@ -1116,12 +1116,40 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
/* negative compression level test : ensure simple API and advanced API produce same result */
DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const srcSize = CNBuffSize / 5;
int const compressionLevel = -1;
assert(cctx != NULL);
{ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0);
size_t const cSize_1pass = ZSTD_compress_advanced(cctx,
compressedBuffer, compressedBufferSize,
CNBuffer, srcSize,
NULL, 0,
params);
if (ZSTD_isError(cSize_1pass)) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel) );
{ ZSTD_inBuffer in = { CNBuffer, srcSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
size_t const compressionResult = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, out.pos);
if (ZSTD_isError(compressionResult)) goto _output_error;
if (out.pos != cSize_1pass) goto _output_error;
} }
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
/* parameters order test */ /* parameters order test */
{ size_t const inputSize = CNBuffSize / 2; { size_t const inputSize = CNBuffSize / 2;
U64 xxh64; U64 xxh64;
{ ZSTD_CCtx* cctx = ZSTD_createCCtx(); { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );

2
tests/paramgrill.c
View File

@ -598,7 +598,7 @@ static ZSTD_compressionParameters randomParams(void)
p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN;
p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN;
p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN;
p.targetLength=(FUZ_rand(&g_rand) % (512)) + ZSTD_TARGETLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512));
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
validated = !ZSTD_isError(ZSTD_checkCParams(p)); validated = !ZSTD_isError(ZSTD_checkCParams(p));
} }