facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #983 from terrelln/dict-wlog

Browse Source 
Increase windowLog from CDict based on the srcSize when known
This commit is contained in:
Yann Collet 2018-01-12 07:47:43 -08:00 committed by GitHub
commit 863b2f8db4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/compress/zstd_compress.c
View File

@ -2498,6 +2498,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
if (cdict==NULL) return ERROR(dictionary_wrong);
{ ZSTD_CCtx_params params = cctx->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict);
/* Increase window log to fit the entire dictionary and source if the
* source size is known. Limit the increase to 19, which is the
* window log for compression level 1 with the largest source size.
*/
if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
params.cParams.windowLog = MAX(params.cParams.windowLog, limitedSrcLog);
}
params.fParams = fParams;
return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto,

22
tests/fuzzer.c
View File

@ -1205,6 +1205,28 @@ static int basicUnitTests(U32 seed, double compressibility)
if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
{
size_t const size = MIN(128 KB, CNBuffSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
ZSTD_frameHeader lgHeader;
ZSTD_frameHeader smHeader;
CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
ZSTD_freeCDict(smCDict);
ZSTD_freeCDict(lgCDict);
ZSTD_freeCCtx(cctx);
}
_end:
free(CNBuffer);
free(compressedBuffer);