adjustCParams : restored previous behavior

unknowns srcSize presumed small if there is a dictionary (dictSize>0)
and presumed large otherwise.
This commit is contained in:
Yann Collet 2017-09-28 18:14:28 -07:00
parent e4ec427720
commit 86b4fe5b45
2 changed files with 23 additions and 20 deletions

View File

@ -99,16 +99,16 @@ static void* POOL_thread(void* opaque) {
/* Unreachable */ /* Unreachable */
} }
POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) { POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
} }
POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
POOL_ctx *ctx; POOL_ctx* ctx;
/* Check the parameters */ /* Check the parameters */
if (!numThreads) { return NULL; } if (!numThreads) { return NULL; }
/* Allocate the context and zero initialize */ /* Allocate the context and zero initialize */
ctx = (POOL_ctx *)ZSTD_calloc(sizeof(POOL_ctx), customMem); ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
if (!ctx) { return NULL; } if (!ctx) { return NULL; }
/* Initialize the job queue. /* Initialize the job queue.
* It needs one extra space since one space is wasted to differentiate empty * It needs one extra space since one space is wasted to differentiate empty
@ -146,7 +146,7 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
/*! POOL_join() : /*! POOL_join() :
Shutdown the queue, wake any sleeping threads, and join all of the threads. Shutdown the queue, wake any sleeping threads, and join all of the threads.
*/ */
static void POOL_join(POOL_ctx *ctx) { static void POOL_join(POOL_ctx* ctx) {
/* Shut down the queue */ /* Shut down the queue */
ZSTD_pthread_mutex_lock(&ctx->queueMutex); ZSTD_pthread_mutex_lock(&ctx->queueMutex);
ctx->shutdown = 1; ctx->shutdown = 1;

View File

@ -640,26 +640,29 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
/** ZSTD_adjustCParams_internal() : /** ZSTD_adjustCParams_internal() :
optimize `cPar` for a given input (`srcSize` and `dictSize`). optimize `cPar` for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization. mostly downsizing to reduce memory consumption and initialization latency.
Both `srcSize` and `dictSize` are optional (use 0 if unknown), Both `srcSize` and `dictSize` are optional (use 0 if unknown).
but if both are 0, no optimization can be done.
Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */ Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */
ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
{ {
static const U64 minSrcSize = 513; /* (1<<9) + 1 */
static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
assert(ZSTD_checkCParams(cPar)==0); assert(ZSTD_checkCParams(cPar)==0);
/* resize windowLog if src is small, to use less memory when necessary */ if (dictSize && (srcSize+1<2) /* srcSize unknown */ )
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (0ULL - 1)); srcSize = minSrcSize; /* presumed small when there is a dictionary */
if ( (dictSize || (srcSize+1 > 1)) /* srcSize test depends on static assert condition */ else
&& (srcSize-1 < (1ULL<<ZSTD_WINDOWLOG_MAX)) ) /* no correction when srcSize is large enough */ { srcSize -= 1; /* unknown 0 => -1ULL : presumed large */
U32 const minSrcSize = (srcSize==0) ? 513 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; /* resize windowLog if input is small enough, to use less memory */
if (rSize < (1ULL<<ZSTD_WINDOWLOG_MAX)) { if ( (srcSize < maxWindowResize)
U32 const srcLog = && (dictSize < maxWindowResize) ) {
MAX(ZSTD_HASHLOG_MIN, U32 const tSize = (U32)(srcSize + dictSize);
(rSize==1) ? 1 : ZSTD_highbit32((U32)(rSize)-1) + 1); static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
} } ZSTD_highbit32(tSize-1) + 1;
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
}
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog) if (cycleLog > cPar.windowLog)