facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add ldmBucketSizeLog param

Browse Source
This commit is contained in:
Stella Lau 2017-09-02 21:10:36 -07:00
parent a1f04d518d
commit 67d4a6161c
9 changed files with 116 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd_internal.h
View File

@ -263,7 +263,7 @@ typedef struct {
typedef struct { typedef struct {
U32 enableLdm; /* 1 if enable long distance matching */ U32 enableLdm; /* 1 if enable long distance matching */
U32 hashLog; /* Log size of hashTable */ U32 hashLog; /* Log size of hashTable */
U32 bucketLog; /* Log number of buckets, at most 4 */ U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */ U32 minMatchLength; /* Minimum match length */
U32 hashEveryLog; /* Log number of entries to skip */ U32 hashEveryLog; /* Log number of entries to skip */
} ldmParams_t; } ldmParams_t;

77
lib/compress/zstd_compress.c
View File

@ -37,7 +37,6 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
#define LDM_BUCKET_SIZE_LOG 3 #define LDM_BUCKET_SIZE_LOG 3
#define LDM_BUCKET_SIZE_LOG_MAX 4
#define LDM_MIN_MATCH_LENGTH 64 #define LDM_MIN_MATCH_LENGTH 64
#define LDM_WINDOW_LOG 27 #define LDM_WINDOW_LOG 27
#define LDM_HASH_LOG 20 #define LDM_HASH_LOG 20
@ -202,6 +201,33 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
+ cctx->outBuffSize + cctx->inBuffSize + cctx->outBuffSize + cctx->inBuffSize
+ ZSTDMT_sizeof_CCtx(cctx->mtctx); + ZSTDMT_sizeof_CCtx(cctx->mtctx);
} }
#if 0
static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params)
{
DEBUGLOG(2, "======CCtxParams======");
DEBUGLOG(2, "cParams: %u %u %u %u %u %u %u",
params->cParams.windowLog,
params->cParams.chainLog,
params->cParams.hashLog,
params->cParams.searchLog,
params->cParams.searchLength,
params->cParams.targetLength,
params->cParams.strategy);
DEBUGLOG(2, "fParams: %u %u %u",
params->fParams.contentSizeFlag,
params->fParams.checksumFlag,
params->fParams.noDictIDFlag);
DEBUGLOG(2, "cLevel, forceWindow: %u %u",
params->compressionLevel,
params->forceWindow);
DEBUGLOG(2, "ldm: %u %u %u %u %u",
params->ldmParams.enableLdm,
params->ldmParams.hashLog,
params->ldmParams.bucketSizeLog,
params->ldmParams.minMatchLength,
params->ldmParams.hashEveryLog);
}
#endif
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{ {
@ -316,10 +342,10 @@ size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThre
static size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm) static size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
{ {
assert(LDM_BUCKET_SIZE_LOG <= LDM_BUCKET_SIZE_LOG_MAX); assert(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
params->enableLdm = enableLdm>0; params->enableLdm = enableLdm>0;
params->hashLog = LDM_HASH_LOG; params->hashLog = LDM_HASH_LOG;
params->bucketLog = LDM_BUCKET_SIZE_LOG; params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
params->minMatchLength = LDM_MIN_MATCH_LENGTH; params->minMatchLength = LDM_MIN_MATCH_LENGTH;
params->hashEveryLog = LDM_HASHEVERYLOG_NOTSET; params->hashEveryLog = LDM_HASHEVERYLOG_NOTSET;
return 0; return 0;
@ -374,7 +400,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->requestedParams.nbThreads); DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->requestedParams.nbThreads);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_longDistanceMatching: case ZSTD_p_enableLongDistanceMatching:
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
if (value != 0) { if (value != 0) {
ZSTD_cLevelToCParams(cctx); ZSTD_cLevelToCParams(cctx);
@ -387,6 +413,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_ldmBucketSizeLog:
case ZSTD_p_ldmHashEveryLog: case ZSTD_p_ldmHashEveryLog:
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@ -490,7 +517,7 @@ size_t ZSTD_CCtxParam_setParameter(
if (params->nbThreads <= 1) return ERROR(parameter_unsupported); if (params->nbThreads <= 1) return ERROR(parameter_unsupported);
return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value); return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value);
case ZSTD_p_longDistanceMatching : case ZSTD_p_enableLongDistanceMatching :
if (value != 0) { if (value != 0) {
ZSTD_cLevelToCCtxParams(params); ZSTD_cLevelToCCtxParams(params);
params->cParams.windowLog = LDM_WINDOW_LOG; params->cParams.windowLog = LDM_WINDOW_LOG;
@ -509,6 +536,13 @@ size_t ZSTD_CCtxParam_setParameter(
params->ldmParams.minMatchLength = value; params->ldmParams.minMatchLength = value;
return 0; return 0;
case ZSTD_p_ldmBucketSizeLog :
if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) {
return ERROR(parameter_outOfBound);
}
params->ldmParams.bucketSizeLog = value;
return 0;
case ZSTD_p_ldmHashEveryLog : case ZSTD_p_ldmHashEveryLog :
if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) { if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) {
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
@ -722,12 +756,11 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
} }
/* Estimate the space needed for long distance matching tables. */ /* Estimate the space needed for long distance matching tables. */
static size_t ZSTD_ldm_getTableSize(U32 ldmHashLog, U32 bucketLog) { static size_t ZSTD_ldm_getTableSize(U32 hashLog, U32 bucketSizeLog) {
size_t const ldmHSize = ((size_t)1) << ldmHashLog; size_t const ldmHSize = ((size_t)1) << hashLog;
size_t const ldmBucketLog = size_t const ldmBucketSizeLog = MIN(bucketSizeLog, hashLog);
MIN(bucketLog, LDM_BUCKET_SIZE_LOG_MAX);
size_t const ldmBucketSize = size_t const ldmBucketSize =
((size_t)1) << (ldmHashLog - ldmBucketLog); ((size_t)1) << (hashLog - ldmBucketSizeLog);
return ldmBucketSize + (ldmHSize * (sizeof(ldmEntry_t))); return ldmBucketSize + (ldmHSize * (sizeof(ldmEntry_t)));
} }
@ -757,7 +790,7 @@ size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* pa
size_t const ldmSpace = params->ldmParams.enableLdm ? size_t const ldmSpace = params->ldmParams.enableLdm ?
ZSTD_ldm_getTableSize(params->ldmParams.hashLog, ZSTD_ldm_getTableSize(params->ldmParams.hashLog,
params->ldmParams.bucketLog) : 0; params->ldmParams.bucketSizeLog) : 0;
size_t const neededSpace = entropySpace + tableSpace + tokenSpace + size_t const neededSpace = entropySpace + tableSpace + tokenSpace +
optSpace + ldmSpace; optSpace + ldmSpace;
@ -824,7 +857,7 @@ static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1,
return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) ||
(ldmParams1.enableLdm == ldmParams2.enableLdm && (ldmParams1.enableLdm == ldmParams2.enableLdm &&
ldmParams1.hashLog == ldmParams2.hashLog && ldmParams1.hashLog == ldmParams2.hashLog &&
ldmParams1.bucketLog == ldmParams2.bucketLog && ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog &&
ldmParams1.minMatchLength == ldmParams2.minMatchLength && ldmParams1.minMatchLength == ldmParams2.minMatchLength &&
ldmParams1.hashEveryLog == ldmParams2.hashEveryLog); ldmParams1.hashEveryLog == ldmParams2.hashEveryLog);
} }
@ -889,11 +922,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
} } } }
if (params.ldmParams.enableLdm) { if (params.ldmParams.enableLdm) {
/* Adjust long distance matching parameters */
if (params.ldmParams.hashEveryLog == LDM_HASHEVERYLOG_NOTSET) { if (params.ldmParams.hashEveryLog == LDM_HASHEVERYLOG_NOTSET) {
params.ldmParams.hashEveryLog = params.ldmParams.hashEveryLog =
params.cParams.windowLog < params.ldmParams.hashLog ? params.cParams.windowLog < params.ldmParams.hashLog ?
0 : params.cParams.windowLog - params.ldmParams.hashLog; 0 : params.cParams.windowLog - params.ldmParams.hashLog;
} }
params.ldmParams.bucketSizeLog =
MIN(params.ldmParams.bucketSizeLog, params.ldmParams.hashLog);
zc->ldmState.hashPower = zc->ldmState.hashPower =
ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength);
} }
@ -915,7 +951,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const ldmSpace = params.ldmParams.enableLdm ? size_t const ldmSpace = params.ldmParams.enableLdm ?
ZSTD_ldm_getTableSize(params.ldmParams.hashLog, ZSTD_ldm_getTableSize(params.ldmParams.hashLog,
params.ldmParams.bucketLog) : 0; params.ldmParams.bucketSizeLog) : 0;
/* Check if workSpace is large enough, alloc a new one if needed */ /* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); { size_t const entropySpace = sizeof(ZSTD_entropyCTables_t);
@ -997,7 +1033,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
if (params.ldmParams.enableLdm) { if (params.ldmParams.enableLdm) {
size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog;
size_t const ldmBucketSize = size_t const ldmBucketSize =
((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketLog); ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog);
memset(ptr, 0, ldmSpace); memset(ptr, 0, ldmSpace);
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->ldmState.hashTable = (ldmEntry_t*)ptr; zc->ldmState.hashTable = (ldmEntry_t*)ptr;
@ -3157,7 +3193,7 @@ static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits)
static ldmEntry_t* ZSTD_ldm_getBucket( static ldmEntry_t* ZSTD_ldm_getBucket(
ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
{ {
return ldmState->hashTable + (hash << ldmParams.bucketLog); return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
} }
/** ZSTD_ldm_insertEntry() : /** ZSTD_ldm_insertEntry() :
@ -3169,7 +3205,7 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
BYTE* const bucketOffsets = ldmState->bucketOffsets; BYTE* const bucketOffsets = ldmState->bucketOffsets;
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry; *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry;
bucketOffsets[hash]++; bucketOffsets[hash]++;
bucketOffsets[hash] &= (1 << ldmParams.bucketLog) - 1; bucketOffsets[hash] &= (1 << ldmParams.bucketSizeLog) - 1;
} }
/** ZSTD_ldm_makeEntryAndInsertByTag() : /** ZSTD_ldm_makeEntryAndInsertByTag() :
@ -3364,8 +3400,8 @@ size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx,
ldmState_t* const ldmState = &(cctx->ldmState); ldmState_t* const ldmState = &(cctx->ldmState);
const ldmParams_t ldmParams = cctx->appliedParams.ldmParams; const ldmParams_t ldmParams = cctx->appliedParams.ldmParams;
const U64 hashPower = ldmState->hashPower; const U64 hashPower = ldmState->hashPower;
const U32 hBits = ldmParams.hashLog - ldmParams.bucketLog; const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog;
const U32 ldmBucketSize = (1 << ldmParams.bucketLog); const U32 ldmBucketSize = (1 << ldmParams.bucketSizeLog);
const U32 ldmTagMask = (1 << ldmParams.hashEveryLog) - 1; const U32 ldmTagMask = (1 << ldmParams.hashEveryLog) - 1;
seqStore_t* const seqStorePtr = &(cctx->seqStore); seqStore_t* const seqStorePtr = &(cctx->seqStore);
const BYTE* const base = cctx->base; const BYTE* const base = cctx->base;
@ -3566,8 +3602,8 @@ static size_t ZSTD_compressBlock_ldm_extDict_generic(
ldmState_t* const ldmState = &(ctx->ldmState); ldmState_t* const ldmState = &(ctx->ldmState);
const ldmParams_t ldmParams = ctx->appliedParams.ldmParams; const ldmParams_t ldmParams = ctx->appliedParams.ldmParams;
const U64 hashPower = ldmState->hashPower; const U64 hashPower = ldmState->hashPower;
const U32 hBits = ldmParams.hashLog - ldmParams.bucketLog; const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog;
const U32 ldmBucketSize = (1 << ldmParams.bucketLog); const U32 ldmBucketSize = (1 << ldmParams.bucketSizeLog);
const U32 ldmTagMask = (1 << ldmParams.hashEveryLog) - 1; const U32 ldmTagMask = (1 << ldmParams.hashEveryLog) - 1;
seqStore_t* const seqStorePtr = &(ctx->seqStore); seqStore_t* const seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base; const BYTE* const base = ctx->base;
@ -5009,7 +5045,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
return flushMin; return flushMin;
} }
#endif #endif
CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) ); CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
DEBUGLOG(5, "completed ZSTD_compress_generic"); DEBUGLOG(5, "completed ZSTD_compress_generic");
return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */

28
lib/zstd.h
View File

@ -392,6 +392,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_LDM_SEARCHLENGTH_MIN 4 #define ZSTD_LDM_SEARCHLENGTH_MIN 4
#define ZSTD_LDM_SEARCHLENGTH_MAX 4096 #define ZSTD_LDM_SEARCHLENGTH_MAX 4096
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
#define ZSTD_FRAMEHEADERSIZE_MIN 6 #define ZSTD_FRAMEHEADERSIZE_MIN 6
@ -978,22 +979,25 @@ typedef enum {
/* advanced parameters - may not remain available after API update */ /* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */
ZSTD_p_longDistanceMatching, /* Enable long distance matching. This ZSTD_p_enableLongDistanceMatching, /* Enable long distance matching. This increases the memory
* increases the memory usage as well as the * usage as well as window size. Note: setting this
* window size. Note: this should be set after * parameter resets all the LDM parameters as well as
* ZSTD_p_compressionLevel and before * ZSTD_p_windowLog. It should be set after
* ZSTD_p_windowLog and other LDM parameters. */ * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and
* other LDM parameters. Setting the compression level
* after this parameter overrides the window log, though LDM
* will remain enabled until explicitly disabled. */
ZSTD_p_ldmHashLog, /* Size of the table for long distance matching. ZSTD_p_ldmHashLog, /* Size of the table for long distance matching.
* Must be clamped between ZSTD_HASHLOG_MIN and * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. */
* ZSTD_HASHLOG_MAX */
ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher. ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
* Must be clamped between ZSTD_LDM_SEARCHLENGTH_MIN * Must be clamped between ZSTD_LDM_SEARCHLENGTH_MIN
* and ZSTD_LDM_SEARCHLENGTH_MAX. */ * and ZSTD_LDM_SEARCHLENGTH_MAX. */
ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the hash table for collision resolution.
* LDM hash table. The default is * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. */
* (windowLog - ldmHashLog) to optimize hash table ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table.
* usage. Must be clamped between 0 and * The default is MAX(0, (windowLog - ldmHashLog)) to
* ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */ * optimize hash table usage.
* Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */
} ZSTD_cParameter; } ZSTD_cParameter;

16
programs/bench.c
View File

@ -144,8 +144,13 @@ void BMK_setLdmHashLog(unsigned ldmHashLog) {
g_ldmHashLog = ldmHashLog; g_ldmHashLog = ldmHashLog;
} }
#define BMK_LDM_HASHEVERYLOG_NOTSET 9999 #define BMK_LDM_PARAM_NOTSET 9999
static U32 g_ldmHashEveryLog = BMK_LDM_HASHEVERYLOG_NOTSET; static U32 g_ldmBucketSizeLog = BMK_LDM_PARAM_NOTSET;
void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
static U32 g_ldmHashEveryLog = BMK_LDM_PARAM_NOTSET;
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) { void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
g_ldmHashEveryLog = ldmHashEveryLog; g_ldmHashEveryLog = ldmHashEveryLog;
} }
@ -286,10 +291,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
#ifdef ZSTD_NEWAPI #ifdef ZSTD_NEWAPI
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_longDistanceMatching, g_ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, g_ldmHashLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, g_ldmHashLog);
if (g_ldmHashEveryLog != BMK_LDM_HASHEVERYLOG_NOTSET) { if (g_ldmBucketSizeLog != BMK_LDM_PARAM_NOTSET) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != BMK_LDM_PARAM_NOTSET) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog);
} }
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);

1
programs/bench.h
View File

@ -28,6 +28,7 @@ void BMK_setDecodeOnlyMode(unsigned decodeFlag);
void BMK_setLdmFlag(unsigned ldmFlag); void BMK_setLdmFlag(unsigned ldmFlag);
void BMK_setLdmMinMatch(unsigned ldmMinMatch); void BMK_setLdmMinMatch(unsigned ldmMinMatch);
void BMK_setLdmHashLog(unsigned ldmHashLog); void BMK_setLdmHashLog(unsigned ldmHashLog);
void BMK_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog); void BMK_setLdmHashEveryLog(unsigned ldmHashEveryLog);
#endif /* BENCH_H_121279284357 */ #endif /* BENCH_H_121279284357 */

18
programs/fileio.c
View File

@ -225,8 +225,14 @@ static U32 g_ldmMinMatch = 0;
void FIO_setLdmMinMatch(unsigned ldmMinMatch) { void FIO_setLdmMinMatch(unsigned ldmMinMatch) {
g_ldmMinMatch = ldmMinMatch; g_ldmMinMatch = ldmMinMatch;
} }
#define FIO_LDM_HASHEVERYLOG_NOTSET 9999
static U32 g_ldmHashEveryLog = FIO_LDM_HASHEVERYLOG_NOTSET; #define FIO_LDM_PARAM_NOTSET 9999
static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET;
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) { void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
g_ldmHashEveryLog = ldmHashEveryLog; g_ldmHashEveryLog = ldmHashEveryLog;
} }
@ -419,10 +425,14 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
/* compression level */ /* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
/* long distance matching */ /* long distance matching */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_longDistanceMatching, g_ldmFlag) ); CHECK( ZSTD_CCtx_setParameter(
ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
if (g_ldmHashEveryLog != FIO_LDM_HASHEVERYLOG_NOTSET) { if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
}
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
} }
/* compression parameters */ /* compression parameters */

1
programs/fileio.h
View File

@ -59,6 +59,7 @@ void FIO_setOverlapLog(unsigned overlapLog);
void FIO_setLdmFlag(unsigned ldmFlag); void FIO_setLdmFlag(unsigned ldmFlag);
void FIO_setLdmHashLog(unsigned ldmHashLog); void FIO_setLdmHashLog(unsigned ldmHashLog);
void FIO_setLdmMinMatch(unsigned ldmMinMatch); void FIO_setLdmMinMatch(unsigned ldmMinMatch);
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog); void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);

18
programs/zstdcli.c
View File

@ -72,11 +72,12 @@ static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 3; static const int g_defaultDictCLevel = 3;
static const unsigned g_defaultSelectivityLevel = 9; static const unsigned g_defaultSelectivityLevel = 9;
#define OVERLAP_LOG_DEFAULT 9999 #define OVERLAP_LOG_DEFAULT 9999
#define LDM_HASHEVERYLOG_DEFAULT 9999 #define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT; static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
static U32 g_ldmHashLog = 0; static U32 g_ldmHashLog = 0;
static U32 g_ldmMinMatch = 0; static U32 g_ldmMinMatch = 0;
static U32 g_ldmHashEveryLog = LDM_HASHEVERYLOG_DEFAULT; static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
/*-************************************ /*-************************************
@ -311,6 +312,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmHlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmHlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmSlen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmSlen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashEveryLog=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "ldmHashEveryLog=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
return 0; return 0;
} }
@ -733,7 +735,12 @@ int main(int argCount, const char* argv[])
BMK_setLdmFlag(ldmFlag); BMK_setLdmFlag(ldmFlag);
BMK_setLdmMinMatch(g_ldmMinMatch); BMK_setLdmMinMatch(g_ldmMinMatch);
BMK_setLdmHashLog(g_ldmHashLog); BMK_setLdmHashLog(g_ldmHashLog);
BMK_setLdmHashEveryLog(g_ldmHashEveryLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
}
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
#endif #endif
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio;
@ -804,7 +811,10 @@ int main(int argCount, const char* argv[])
FIO_setLdmFlag(ldmFlag); FIO_setLdmFlag(ldmFlag);
FIO_setLdmHashLog(g_ldmHashLog); FIO_setLdmHashLog(g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch); FIO_setLdmMinMatch(g_ldmMinMatch);
if (g_ldmHashEveryLog != LDM_HASHEVERYLOG_DEFAULT) { if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
FIO_setLdmHashEveryLog(g_ldmHashEveryLog); FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
} }

2
tests/zstreamtest.c
View File

@ -1380,7 +1380,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_longDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
/* unconditionally set, to be sync with decoder */ /* unconditionally set, to be sync with decoder */
/* mess with frame parameters */ /* mess with frame parameters */