facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improved ZSTDMT_compress() memory usage

Browse Source 
does not need the input buffer for streaming operations

also : reduced a few tests time length
This commit is contained in:
Yann Collet 2017-07-10 17:16:41 -07:00
parent 670b1fc547
commit 4616fad18b
4 changed files with 36 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
lib/compress/zstdmt_compress.c
View File

@ -134,9 +134,11 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
/* large enough, but not too much */
return buf;
/* size conditions not respected : scratch this buffer, create new one */
DEBUGLOG(2, "existing buffer does not meet size conditions => freeing");
ZSTD_free(buf.start, pool->cMem);
}
/* create new buffer */
DEBUGLOG(2, "create a new buffer");
{ buffer_t buffer;
void* const start = ZSTD_malloc(bSize, pool->cMem);
if (start==NULL) bSize = 0;
@ -149,12 +151,14 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
/* store buffer for later re-use, up to pool capacity */
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
{
DEBUGLOG(2, "ZSTDMT_releaseBuffer");
if (buf.start == NULL) return; /* release on NULL */
if (pool->nbBuffers < pool->totalBuffers) {
pool->bTable[pool->nbBuffers++] = buf; /* store for later re-use */
return;
}
/* Reached bufferPool capacity (should not happen) */
DEBUGLOG(2, "buffer pool capacity reached => freeing ");
ZSTD_free(buf.start, pool->cMem);
}
@ -635,8 +639,8 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
if (zcs->nbThreads==1) {
DEBUGLOG(4, "single thread mode");
return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
dict, dictSize, cdict,
params, pledgedSrcSize);
dict, dictSize, cdict,
params, pledgedSrcSize);
}
if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */
@ -671,9 +675,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
zcs->marginSize = zcs->targetSectionSize >> 2;
zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
zcs->inBuff.filled = 0;
zcs->inBuff.buffer = g_nullBuffer;
zcs->dictSize = 0;
zcs->doneJobID = 0;
zcs->nextJobID = 0;
@ -899,13 +901,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
}
/* fill input buffer */
if ((input->src) && (mtctx->inBuff.buffer.start)) { /* support NULL input */
size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
DEBUGLOG(5, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
input->pos += toLoad;
mtctx->inBuff.filled += toLoad;
}
if (input->src) { /* support NULL input */
if (mtctx->inBuff.buffer.start == NULL) {
mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize);
if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
mtctx->inBuff.filled = 0;
}
{ size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
DEBUGLOG(5, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
input->pos += toLoad;
mtctx->inBuff.filled += toLoad;
} }
if ( (mtctx->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */
&& (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) { /* avoid overwriting job round buffer */

2
tests/fuzzer.c
View File

@ -129,7 +129,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
static void FUZ_freeDebug(void* counter, void* address)
{
mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
DISPLAYLEVEL(4, "releasing %u KB \n", (U32)(malloc_size(address) >> 10));
DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10));
mcPtr->nbFree += 1;
mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
free(address);

31
tests/playTests.sh
View File

@ -7,17 +7,17 @@ die() {
roundTripTest() {
if [ -n "$3" ]; then
local_c="$3"
local_p="$2"
cLevel="$3"
proba="$2"
else
local_c="$2"
local_p=""
cLevel="$2"
proba=""
fi
rm -f tmp1 tmp2
$ECHO "roundTripTest: ./datagen $1 $local_p | $ZSTD -v$local_c | $ZSTD -d"
./datagen $1 $local_p | $MD5SUM > tmp1
./datagen $1 $local_p | $ZSTD --ultra -v$local_c | $ZSTD -d | $MD5SUM > tmp2
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
./datagen $1 $proba | $MD5SUM > tmp1
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
@ -625,16 +625,15 @@ roundTripTest -g35000000 -P75 10
roundTripTest -g35000000 -P75 11
roundTripTest -g35000000 -P75 12
roundTripTest -g18000000 -P80 13
roundTripTest -g18000000 -P80 14
roundTripTest -g18000000 -P80 15
roundTripTest -g18000000 -P80 16
roundTripTest -g18000000 -P80 17
roundTripTest -g18000013 -P80 13
roundTripTest -g18000014 -P80 14
roundTripTest -g18000015 -P80 15
roundTripTest -g18000016 -P80 16
roundTripTest -g18000017 -P80 17
roundTripTest -g18000018 -P94 18
roundTripTest -g18000019 -P94 19
roundTripTest -g50000000 -P94 18
roundTripTest -g50000000 -P94 19
roundTripTest -g99000000 -P99 20
roundTripTest -g68000020 -P99 20
roundTripTest -g6000000000 -P99 1
fileRoundTripTest -g4193M -P99 1

19
tests/zstreamtest.c
View File

@ -95,19 +95,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
return rand32 >> 5;
}
static void* allocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
return address;
}
static void freeFunction(void* opaque, void* address)
{
(void)opaque;
free(address);
}
/*======================================================
* Basic Unit tests
@ -1543,7 +1530,6 @@ int main(int argc, const char** argv)
int bigTests = (sizeof(size_t) == 8);
e_api selected_api = simple_api;
const char* const programName = argv[0];
ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
ZSTD_customMem const customNULL = ZSTD_defaultCMem;
/* Check command line */
@ -1657,10 +1643,7 @@ int main(int argc, const char** argv)
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result) {
DISPLAYLEVEL(3, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
}
if (!result) {
switch(selected_api)