facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ensure adjustCParams adjust hLog and cLog even without srcSize

Browse Source 
It would previously exit when srcSize is unknown.
But in the case of custom parameters,
hLog and cLog can still be too large in comparison with windowLog.

Reduces maximum memory allocated during zstreamtest --newapi
This commit is contained in:
Yann Collet 2017-09-28 01:25:40 -07:00
parent aa800c4793
commit 9b5b47ac93
3 changed files with 42 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/compress/zstd_compress.c
View File

@ -634,26 +634,31 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
mostly downsizing to reduce memory consumption and initialization. mostly downsizing to reduce memory consumption and initialization.
Both `srcSize` and `dictSize` are optional (use 0 if unknown), Both `srcSize` and `dictSize` are optional (use 0 if unknown),
but if both are 0, no optimization can be done. but if both are 0, no optimization can be done.
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */
ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
{ {
assert(ZSTD_checkCParams(cPar)==0); assert(ZSTD_checkCParams(cPar)==0);
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
/* resize params, to use less memory when necessary */ /* resize windowLog if src is small, to use less memory when necessary */
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == -1ULL);
if ( (dictSize || (srcSize+1 > 1)) /* srcSize test depends on static assert condition */
&& (srcSize-1 < (1ULL<<ZSTD_WINDOWLOG_MAX)) ) /* no correction is srcSize is large enough */ {
U32 const minSrcSize = (srcSize==0) ? 513 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { if (rSize < (1ULL<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = U32 const srcLog =
MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1); MAX(ZSTD_HASHLOG_MIN,
(rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} } } }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); if (cycleLog > cPar.windowLog)
cPar.chainLog -= (cycleLog - cPar.windowLog);
} }
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
return cPar; return cPar;
} }

4
lib/zstd.h
View File

@ -389,8 +389,8 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MIN 4 /* only useful for btopt */
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999 /* only useful for btopt */
#define ZSTD_LDM_MINMATCH_MIN 4 #define ZSTD_LDM_MINMATCH_MIN 4
#define ZSTD_LDM_MINMATCH_MAX 4096 #define ZSTD_LDM_MINMATCH_MAX 4096
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8

17
tests/zstreamtest.c
View File

@ -1322,6 +1322,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
else { DISPLAYUPDATE(2, "\r%6u ", testNb); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed); FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime32; lseed = coreSeed ^ prime32;
DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
/* states full reset (deliberately not synchronized) */ /* states full reset (deliberately not synchronized) */
/* some issues can only happen when reusing states */ /* some issues can only happen when reusing states */
@ -1371,7 +1372,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
(MAX(testLog, dictLog) / 3))) + (MAX(testLog, dictLog) / 3))) +
1; 1;
U32 const cLevel = MIN(cLevelCandidate, cLevelMax); U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
maxTestSize = FUZ_rLogLength(&lseed, testLog); maxTestSize = FUZ_rLogLength(&lseed, testLog);
DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
oldTestLog = testLog; oldTestLog = testLog;
/* random dictionary selection */ /* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
@ -1396,9 +1399,16 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) ); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
}
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
}
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
} }
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
@ -1415,13 +1425,14 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize); DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
/* multi-threading parameters */ /* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) ); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
if (nbThreads > 1) { if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) ); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );