ensure adjustCParams adjust hLog and cLog even without srcSize
It would previously exit when srcSize is unknown. But in the case of custom parameters, hLog and cLog can still be too large in comparison with windowLog. Reduces maximum memory allocated during zstreamtest --newapidev
Yann Collet
parent
aa800c4793
commit
9b5b47ac93
|
|
@ -634,26 +634,31 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
|
|||
mostly downsizing to reduce memory consumption and initialization.
|
||||
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
||||
but if both are 0, no optimization can be done.
|
||||
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
||||
Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */
|
||||
ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
|
||||
{
|
||||
assert(ZSTD_checkCParams(cPar)==0);
|
||||
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
|
||||
|
||||
/* resize params, to use less memory when necessary */
|
||||
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
||||
/* resize windowLog if src is small, to use less memory when necessary */
|
||||
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == -1ULL);
|
||||
if ( (dictSize || (srcSize+1 > 1)) /* srcSize test depends on static assert condition */
|
||||
&& (srcSize-1 < (1ULL<<ZSTD_WINDOWLOG_MAX)) ) /* no correction is srcSize is large enough */ {
|
||||
U32 const minSrcSize = (srcSize==0) ? 513 : 0;
|
||||
U64 const rSize = srcSize + dictSize + minSrcSize;
|
||||
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
||||
if (rSize < (1ULL<<ZSTD_WINDOWLOG_MAX)) {
|
||||
U32 const srcLog =
|
||||
MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
|
||||
MAX(ZSTD_HASHLOG_MIN,
|
||||
(rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1);
|
||||
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
|
||||
} }
|
||||
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
|
||||
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
|
||||
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
|
||||
if (cycleLog > cPar.windowLog)
|
||||
cPar.chainLog -= (cycleLog - cPar.windowLog);
|
||||
}
|
||||
|
||||
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
|
||||
cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||
|
||||
return cPar;
|
||||
}
|
||||
|
|
|
|||
30
lib/zstd.h
30
lib/zstd.h
|
|
@ -376,22 +376,22 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
|||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX_32 30
|
||||
#define ZSTD_WINDOWLOG_MAX_64 31
|
||||
#define ZSTD_WINDOWLOG_MAX_32 30
|
||||
#define ZSTD_WINDOWLOG_MAX_64 31
|
||||
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
||||
#define ZSTD_WINDOWLOG_MIN 10
|
||||
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
|
||||
#define ZSTD_HASHLOG_MIN 6
|
||||
#define ZSTD_CHAINLOG_MAX MIN(ZSTD_WINDOWLOG_MAX+1, 30)
|
||||
#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
|
||||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
|
||||
#define ZSTD_TARGETLENGTH_MIN 4
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
#define ZSTD_LDM_MINMATCH_MIN 4
|
||||
#define ZSTD_WINDOWLOG_MIN 10
|
||||
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
|
||||
#define ZSTD_HASHLOG_MIN 6
|
||||
#define ZSTD_CHAINLOG_MAX MIN(ZSTD_WINDOWLOG_MAX+1, 30)
|
||||
#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
|
||||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
|
||||
#define ZSTD_TARGETLENGTH_MIN 4 /* only useful for btopt */
|
||||
#define ZSTD_TARGETLENGTH_MAX 999 /* only useful for btopt */
|
||||
#define ZSTD_LDM_MINMATCH_MIN 4
|
||||
#define ZSTD_LDM_MINMATCH_MAX 4096
|
||||
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
|
||||
|
||||
|
|
|
|||
|
|
@ -1322,6 +1322,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
|||
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime32;
|
||||
DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
|
||||
|
||||
/* states full reset (deliberately not synchronized) */
|
||||
/* some issues can only happen when reusing states */
|
||||
|
|
@ -1371,7 +1372,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
|||
(MAX(testLog, dictLog) / 3))) +
|
||||
1;
|
||||
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
|
||||
DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
|
||||
oldTestLog = testLog;
|
||||
/* random dictionary selection */
|
||||
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
|
||||
|
|
@ -1396,9 +1399,16 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
|||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
|
||||
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
|
||||
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
|
||||
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
|
||||
|
|
@ -1415,13 +1425,14 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
|||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
|
||||
DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
|
||||
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
|
||||
|
||||
/* multi-threading parameters */
|
||||
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
|
||||
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
|
||||
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
|
||||
if (nbThreads > 1) {
|
||||
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );
|
||||
|
|
|
|||
Loading…
Reference in New Issue