facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes adaptation on srcSize 

This patch restores capability for each file to receive adapted compression parameters depending on its size.

The bug breaking this feature was relatively silly :
setting a parameter with a value "0" is supposed to be a no-op.
Unfortunately, it would pin down compression parameters as if they were manually set,
preventing later automatic adaptation.

Unfortunately, I'm currently short of a test case that could check this situation and trigger an error.
Compression parameters selection between tableID 0,1,2,3 is largely internal,
leaving no trace to outside world, not even in frame header.
dev
Yann Collet 2017-12-13 17:45:26 -08:00
parent 90d38f6a53
commit 5432ef6921
2 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/compress/zstd_compress.c
View File

@ -144,6 +144,8 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize) ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize)
{ {
DEBUGLOG(4, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u",
(U32)srcSizeHint, (U32)dictSize);
return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ? return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ?
CCtxParams.cParams : CCtxParams.cParams :
ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize); ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize);
@ -151,18 +153,22 @@ static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize) static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize)
{ {
DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u",
(U32)srcSize);
CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0); CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0);
CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
} }
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{ {
DEBUGLOG(4, "ZSTD_cLevelToCParams");
ZSTD_cLevelToCCtxParams_srcSize( ZSTD_cLevelToCCtxParams_srcSize(
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
} }
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams) static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
{ {
DEBUGLOG(4, "ZSTD_cLevelToCCtxParams");
ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, 0); ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, 0);
} }
@ -261,7 +267,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
case ZSTD_p_targetLength: case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy: case ZSTD_p_compressionStrategy:
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */ if (value>0) ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_contentSizeFlag: case ZSTD_p_contentSizeFlag:
@ -422,7 +428,7 @@ size_t ZSTD_CCtxParam_setParameter(
#endif #endif
case ZSTD_p_enableLongDistanceMatching : case ZSTD_p_enableLongDistanceMatching :
if (value != 0) { if (value) {
ZSTD_cLevelToCCtxParams(CCtxParams); ZSTD_cLevelToCCtxParams(CCtxParams);
CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
} }
@ -3056,6 +3062,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
size_t const addedSize = srcSizeHint ? 0 : 500; size_t const addedSize = srcSizeHint ? 0 : 500;
U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1; U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
DEBUGLOG(4, "ZSTD_getCParams: cLevel=%i, srcSize=%u, dictSize=%u => table %u",
compressionLevel, (U32)srcSizeHint, (U32)dictSize, tableID);
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
static int g_monotonicTest = 1; static int g_monotonicTest = 1;

2
programs/fileio.c
View File

@ -446,7 +446,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, using good compression parameters */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, for compression parameters adaptation */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */