facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2366 from senhuang42/enable_ldm_by_default

Browse Source 
Enable LDM by default if window size >= 128MB and strategy uses opt parser
This commit is contained in:
sen 2020-10-27 14:59:28 -04:00 committed by GitHub
commit 17b700d78a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 46 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/compress/zstd_compress.c
View File

@ -202,6 +202,14 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
/* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
/* Returns 1 if compression parameters are such that we should
* enable long distance matching (wlog >= 27, strategy >= btopt).
* Returns 0 otherwise.
*/
static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const cParams) {
return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27;
}
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_compressionParameters cParams)
{
@ -209,6 +217,16 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
/* should not matter, as all cParams are presumed properly defined */
ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT);
cctxParams.cParams = cParams;
if (ZSTD_CParams_shouldEnableLdm(&cParams)) {
DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params");
cctxParams.ldmParams.enableLdm = 1;
/* LDM is enabled by default for optimal parser and window size >= 128MB */
ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams);
assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog);
assert(cctxParams.ldmParams.hashRateLog < 32);
}
assert(!ZSTD_checkCParams(cParams));
return cctxParams;
}
@ -4183,6 +4201,11 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
dictSize, mode);
}
if (ZSTD_CParams_shouldEnableLdm(&params.cParams)) {
/* Enable LDM by default for optimal parser and window size >= 128MB */
DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)");
params.ldmParams.enableLdm = 1;
}
#ifdef ZSTD_MULTITHREAD
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {

4
lib/zstd.h
View File

@ -339,7 +339,9 @@ typedef enum {
* for large inputs, by finding large matches at long distance.
* It increases memory usage and window size.
* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
* except when expressly set to a different value. */
* except when expressly set to a different value.
* Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
* compression strategy >= ZSTD_btopt (== compression level 16+) */
ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
* Larger values increase memory usage and compression ratio,
* but decrease compression speed.

20
tests/fuzzer.c
View File

@ -1158,6 +1158,26 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_outBuffer out = {NULL, 0, 0};
ZSTD_inBuffer in = {NULL, 0, 0};
int value;
/* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
* we should not modify the actual parameter specified by the user within the CCtx
*/
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
CHECK_EQ(value, 0);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
/* this test is really too long, and should be made faster */
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();