facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[libzstd] Clean up parameter code

Browse Source 
* Move all ZSTDMT parameter setting code to ZSTD_CCtxParams_*Parameter().
  ZSTDMT now calls these functions, so we can keep all the logic in the
  same place.
* Clean up `ZSTD_CCtx_setParameter()` to only add extra checks where needed.
* Clean up `ZSTDMT_initJobCCtxParams()` by copying all parameters by default,
  and then zeroing the ones that need to be zeroed. We've missed adding several
  parameters here, and it makes more sense to only have to update it if you
  change something in ZSTDMT.
* Add `ZSTDMT_cParam_clampBounds()` to clamp a parameter into its valid
  range. Use this to keep backwards compatibility when setting ZSTDMT parameters,
  which clamp into the valid range.
This commit is contained in:
Nick Terrell 2019-02-15 16:15:20 -08:00
parent 54e9412ddd
commit f4abba02ba
4 changed files with 599 additions and 578 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
lib/compress/zstd_compress.c
View File

@ -385,6 +385,18 @@ static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
return 1; return 1;
} }
/* ZSTD_cParam_clampBounds:
* Clamps the value into the bounded range.
*/
static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value)
{
ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
if (ZSTD_isError(bounds.error)) return bounds.error;
if (*value < bounds.lowerBound) *value = bounds.lowerBound;
if (*value > bounds.upperBound) *value = bounds.upperBound;
return 0;
}
#define BOUNDCHECK(cParam, val) { \ #define BOUNDCHECK(cParam, val) { \
RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
parameter_outOfBound); \ parameter_outOfBound); \
@ -438,13 +450,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
switch(param) switch(param)
{ {
case ZSTD_c_format :
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_compressionLevel: case ZSTD_c_compressionLevel:
RETURN_ERROR_IF(cctx->cdict, stage_wrong, RETURN_ERROR_IF(cctx->cdict, stage_wrong,
"compression level is configured in cdict"); "compression level is configured in cdict");
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); break;
case ZSTD_c_windowLog: case ZSTD_c_windowLog:
case ZSTD_c_hashLog: case ZSTD_c_hashLog:
@ -455,44 +464,37 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_strategy: case ZSTD_c_strategy:
RETURN_ERROR_IF(cctx->cdict, stage_wrong, RETURN_ERROR_IF(cctx->cdict, stage_wrong,
"cparams are configured in cdict"); "cparams are configured in cdict");
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); break;
case ZSTD_c_contentSizeFlag:
case ZSTD_c_checksumFlag:
case ZSTD_c_dictIDFlag:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_forceAttachDict:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_literalCompressionMode:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_nbWorkers: case ZSTD_c_nbWorkers:
RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
"MT not compatible with static alloc"); "MT not compatible with static alloc");
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); break;
case ZSTD_c_ldmHashRateLog:
RETURN_ERROR_IF(cctx->cdict, stage_wrong,
"LDM hash rate log is configured in cdict");
break;
case ZSTD_c_format:
case ZSTD_c_contentSizeFlag:
case ZSTD_c_checksumFlag:
case ZSTD_c_dictIDFlag:
case ZSTD_c_forceMaxWindow:
case ZSTD_c_forceAttachDict:
case ZSTD_c_literalCompressionMode:
case ZSTD_c_jobSize: case ZSTD_c_jobSize:
case ZSTD_c_overlapLog: case ZSTD_c_overlapLog:
case ZSTD_c_rsyncable: case ZSTD_c_rsyncable:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_enableLongDistanceMatching: case ZSTD_c_enableLongDistanceMatching:
case ZSTD_c_ldmHashLog: case ZSTD_c_ldmHashLog:
case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmMinMatch:
case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_ldmBucketSizeLog:
case ZSTD_c_ldmHashRateLog: break;
RETURN_ERROR_IF(cctx->cdict, stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
default: RETURN_ERROR(parameter_unsupported); default: RETURN_ERROR(parameter_unsupported);
} }
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
} }
size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
@ -507,11 +509,9 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
return (size_t)CCtxParams->format; return (size_t)CCtxParams->format;
case ZSTD_c_compressionLevel : { case ZSTD_c_compressionLevel : {
int cLevel = value; FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (value) { /* 0 : does not change current level */
if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel(); CCtxParams->compressionLevel = value;
if (cLevel) { /* 0 : does not change current level */
CCtxParams->compressionLevel = cLevel;
} }
if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel; if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel;
return 0; /* return type (size_t) cannot represent negative values */ return 0; /* return type (size_t) cannot represent negative values */
@ -597,28 +597,43 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
return 0; return 0;
#else #else
return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
CCtxParams->nbWorkers = value;
return CCtxParams->nbWorkers;
#endif #endif
case ZSTD_c_jobSize : case ZSTD_c_jobSize :
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
return 0;
#else #else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value); /* Adjust to the minimum non-default value. */
if (value != 0 && value < ZSTDMT_JOBSIZE_MIN)
value = ZSTDMT_JOBSIZE_MIN;
FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value));
assert(value >= 0);
CCtxParams->jobSize = value;
return CCtxParams->jobSize;
#endif #endif
case ZSTD_c_overlapLog : case ZSTD_c_overlapLog :
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
return 0;
#else #else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value); FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value));
CCtxParams->overlapLog = value;
return CCtxParams->overlapLog;
#endif #endif
case ZSTD_c_rsyncable : case ZSTD_c_rsyncable :
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
return 0;
#else #else
return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value); FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value));
CCtxParams->rsyncable = value;
return CCtxParams->rsyncable;
#endif #endif
case ZSTD_c_enableLongDistanceMatching : case ZSTD_c_enableLongDistanceMatching :

54
lib/compress/zstdmt_compress.c
View File

@ -864,11 +864,7 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) {
* Internal use only */ * Internal use only */
size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers)
{ {
if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX; return ZSTD_CCtxParam_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers);
params->nbWorkers = nbWorkers;
params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->jobSize = 0;
return nbWorkers;
} }
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem)
@ -986,26 +982,13 @@ ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
{ {
case ZSTDMT_p_jobSize : case ZSTDMT_p_jobSize :
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
if ( value != 0 /* default */ return ZSTD_CCtxParam_setParameter(params, ZSTD_c_jobSize, value);
&& value < ZSTDMT_JOBSIZE_MIN)
value = ZSTDMT_JOBSIZE_MIN;
assert(value >= 0);
if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX;
params->jobSize = value;
return value;
case ZSTDMT_p_overlapLog : case ZSTDMT_p_overlapLog :
DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value); DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN; return ZSTD_CCtxParam_setParameter(params, ZSTD_c_overlapLog, value);
if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
params->overlapLog = value;
return value;
case ZSTDMT_p_rsyncable : case ZSTDMT_p_rsyncable :
value = (value != 0); DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value);
params->rsyncable = value; return ZSTD_CCtxParam_setParameter(params, ZSTD_c_rsyncable, value);
return value;
default : default :
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
} }
@ -1021,32 +1004,29 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter,
{ {
switch (parameter) { switch (parameter) {
case ZSTDMT_p_jobSize: case ZSTDMT_p_jobSize:
assert(mtctx->params.jobSize <= INT_MAX); return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_jobSize, value);
*value = (int)(mtctx->params.jobSize);
break;
case ZSTDMT_p_overlapLog: case ZSTDMT_p_overlapLog:
*value = mtctx->params.overlapLog; return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_overlapLog, value);
break;
case ZSTDMT_p_rsyncable: case ZSTDMT_p_rsyncable:
*value = mtctx->params.rsyncable; return ZSTD_CCtxParam_getParameter(&mtctx->params, ZSTD_c_rsyncable, value);
break;
default: default:
return ERROR(parameter_unsupported); return ERROR(parameter_unsupported);
} }
return 0;
} }
/* Sets parameters relevant to the compression job, /* Sets parameters relevant to the compression job,
* initializing others to default values. */ * initializing others to default values. */
static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params)
{ {
ZSTD_CCtx_params jobParams; ZSTD_CCtx_params jobParams = params;
memset(&jobParams, 0, sizeof(jobParams)); /* Clear parameters related to multithreading */
jobParams.forceWindow = 0;
jobParams.cParams = params.cParams; jobParams.nbWorkers = 0;
jobParams.fParams = params.fParams; jobParams.jobSize = 0;
jobParams.compressionLevel = params.compressionLevel; jobParams.overlapLog = 0;
jobParams.rsyncable = 0;
memset(&jobParams.ldmParams, 0, sizeof(ldmParams_t));
memset(&jobParams.customMem, 0, sizeof(ZSTD_customMem));
return jobParams; return jobParams;
} }

12
tests/regression/config.c
View File

@ -90,6 +90,17 @@ static config_t mt_ldm = {
.param_values = PARAM_VALUES(mt_ldm_param_values), .param_values = PARAM_VALUES(mt_ldm_param_values),
}; };
static param_value_t mt_advanced_param_values[] = {
{.param = ZSTD_c_nbWorkers, .value = 2},
{.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed},
};
static config_t mt_advanced = {
.name = "multithreaded with advanced params",
.cli_args = "-T2 --no-compressed-literals",
.param_values = PARAM_VALUES(mt_advanced_param_values),
};
static param_value_t const small_wlog_param_values[] = { static param_value_t const small_wlog_param_values[] = {
{.param = ZSTD_c_windowLog, .value = 10}, {.param = ZSTD_c_windowLog, .value = 10},
}; };
@ -191,6 +202,7 @@ static config_t const* g_configs[] = {
&uncompressed_literals, &uncompressed_literals,
&uncompressed_literals_opt, &uncompressed_literals_opt,
&huffman_literals, &huffman_literals,
&mt_advanced,
NULL, NULL,
}; };

14
tests/regression/results.csv
View File

@ -40,6 +40,7 @@ silesia, explicit params, compress cct
silesia, uncompressed literals, compress cctx, 4862377 silesia, uncompressed literals, compress cctx, 4862377
silesia, uncompressed literals optimal, compress cctx, 4293262 silesia, uncompressed literals optimal, compress cctx, 4293262
silesia, huffman literals, compress cctx, 6191548 silesia, huffman literals, compress cctx, 6191548
silesia, multithreaded with advanced params, compress cctx, 4862377
github, level -5, compress cctx, 232744 github, level -5, compress cctx, 232744
github, level -5 with dict, compress cctx, 47294 github, level -5 with dict, compress cctx, 47294
github, level -3, compress cctx, 220611 github, level -3, compress cctx, 220611
@ -78,6 +79,7 @@ github, explicit params, compress cct
github, uncompressed literals, compress cctx, 136397 github, uncompressed literals, compress cctx, 136397
github, uncompressed literals optimal, compress cctx, 133717 github, uncompressed literals optimal, compress cctx, 133717
github, huffman literals, compress cctx, 176575 github, huffman literals, compress cctx, 176575
github, multithreaded with advanced params, compress cctx, 141473
silesia, level -5, zstdcli, 7152342 silesia, level -5, zstdcli, 7152342
silesia, level -3, zstdcli, 6790021 silesia, level -3, zstdcli, 6790021
silesia, level -1, zstdcli, 6191597 silesia, level -1, zstdcli, 6191597
@ -102,6 +104,7 @@ silesia, explicit params, zstdcli,
silesia, uncompressed literals, zstdcli, 5155472 silesia, uncompressed literals, zstdcli, 5155472
silesia, uncompressed literals optimal, zstdcli, 4325475 silesia, uncompressed literals optimal, zstdcli, 4325475
silesia, huffman literals, zstdcli, 5341405 silesia, huffman literals, zstdcli, 5341405
silesia, multithreaded with advanced params, zstdcli, compression error
silesia.tar, level -5, zstdcli, 7161160 silesia.tar, level -5, zstdcli, 7161160
silesia.tar, level -3, zstdcli, 6789865 silesia.tar, level -3, zstdcli, 6789865
silesia.tar, level -1, zstdcli, 6196433 silesia.tar, level -1, zstdcli, 6196433
@ -127,6 +130,7 @@ silesia.tar, explicit params, zstdcli,
silesia.tar, uncompressed literals, zstdcli, 5158134 silesia.tar, uncompressed literals, zstdcli, 5158134
silesia.tar, uncompressed literals optimal, zstdcli, 4321098 silesia.tar, uncompressed literals optimal, zstdcli, 4321098
silesia.tar, huffman literals, zstdcli, 5358479 silesia.tar, huffman literals, zstdcli, 5358479
silesia.tar, multithreaded with advanced params, zstdcli, compression error
github, level -5, zstdcli, 234744 github, level -5, zstdcli, 234744
github, level -5 with dict, zstdcli, 48718 github, level -5 with dict, zstdcli, 48718
github, level -3, zstdcli, 222611 github, level -3, zstdcli, 222611
@ -165,6 +169,7 @@ github, explicit params, zstdcli,
github, uncompressed literals, zstdcli, 169004 github, uncompressed literals, zstdcli, 169004
github, uncompressed literals optimal, zstdcli, 158824 github, uncompressed literals optimal, zstdcli, 158824
github, huffman literals, zstdcli, 145457 github, huffman literals, zstdcli, 145457
github, multithreaded with advanced params, zstdcli, compression error
silesia, level -5, advanced one pass, 7152294 silesia, level -5, advanced one pass, 7152294
silesia, level -3, advanced one pass, 6789969 silesia, level -3, advanced one pass, 6789969
silesia, level -1, advanced one pass, 6191548 silesia, level -1, advanced one pass, 6191548
@ -190,6 +195,7 @@ silesia, explicit params, advanced one
silesia, uncompressed literals, advanced one pass, 5155424 silesia, uncompressed literals, advanced one pass, 5155424
silesia, uncompressed literals optimal, advanced one pass, 4325427 silesia, uncompressed literals optimal, advanced one pass, 4325427
silesia, huffman literals, advanced one pass, 5341356 silesia, huffman literals, advanced one pass, 5341356
silesia, multithreaded with advanced params, advanced one pass, 5155424
silesia.tar, level -5, advanced one pass, 7160438 silesia.tar, level -5, advanced one pass, 7160438
silesia.tar, level -3, advanced one pass, 6789024 silesia.tar, level -3, advanced one pass, 6789024
silesia.tar, level -1, advanced one pass, 6195462 silesia.tar, level -1, advanced one pass, 6195462
@ -215,6 +221,7 @@ silesia.tar, explicit params, advanced one
silesia.tar, uncompressed literals, advanced one pass, 5157992 silesia.tar, uncompressed literals, advanced one pass, 5157992
silesia.tar, uncompressed literals optimal, advanced one pass, 4321094 silesia.tar, uncompressed literals optimal, advanced one pass, 4321094
silesia.tar, huffman literals, advanced one pass, 5358079 silesia.tar, huffman literals, advanced one pass, 5358079
silesia.tar, multithreaded with advanced params, advanced one pass, 5158545
github, level -5, advanced one pass, 232744 github, level -5, advanced one pass, 232744
github, level -5 with dict, advanced one pass, 46718 github, level -5 with dict, advanced one pass, 46718
github, level -3, advanced one pass, 220611 github, level -3, advanced one pass, 220611
@ -254,6 +261,7 @@ github, explicit params, advanced one
github, uncompressed literals, advanced one pass, 167004 github, uncompressed literals, advanced one pass, 167004
github, uncompressed literals optimal, advanced one pass, 156824 github, uncompressed literals optimal, advanced one pass, 156824
github, huffman literals, advanced one pass, 143457 github, huffman literals, advanced one pass, 143457
github, multithreaded with advanced params, advanced one pass, 167004
silesia, level -5, advanced one pass small out, 7152294 silesia, level -5, advanced one pass small out, 7152294
silesia, level -3, advanced one pass small out, 6789969 silesia, level -3, advanced one pass small out, 6789969
silesia, level -1, advanced one pass small out, 6191548 silesia, level -1, advanced one pass small out, 6191548
@ -279,6 +287,7 @@ silesia, explicit params, advanced one
silesia, uncompressed literals, advanced one pass small out, 5155424 silesia, uncompressed literals, advanced one pass small out, 5155424
silesia, uncompressed literals optimal, advanced one pass small out, 4325427 silesia, uncompressed literals optimal, advanced one pass small out, 4325427
silesia, huffman literals, advanced one pass small out, 5341356 silesia, huffman literals, advanced one pass small out, 5341356
silesia, multithreaded with advanced params, advanced one pass small out, 5155424
silesia.tar, level -5, advanced one pass small out, 7160438 silesia.tar, level -5, advanced one pass small out, 7160438
silesia.tar, level -3, advanced one pass small out, 6789024 silesia.tar, level -3, advanced one pass small out, 6789024
silesia.tar, level -1, advanced one pass small out, 6195462 silesia.tar, level -1, advanced one pass small out, 6195462
@ -304,6 +313,7 @@ silesia.tar, explicit params, advanced one
silesia.tar, uncompressed literals, advanced one pass small out, 5157992 silesia.tar, uncompressed literals, advanced one pass small out, 5157992
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321094 silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321094
silesia.tar, huffman literals, advanced one pass small out, 5358079 silesia.tar, huffman literals, advanced one pass small out, 5358079
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5158545
github, level -5, advanced one pass small out, 232744 github, level -5, advanced one pass small out, 232744
github, level -5 with dict, advanced one pass small out, 46718 github, level -5 with dict, advanced one pass small out, 46718
github, level -3, advanced one pass small out, 220611 github, level -3, advanced one pass small out, 220611
@ -343,6 +353,7 @@ github, explicit params, advanced one
github, uncompressed literals, advanced one pass small out, 167004 github, uncompressed literals, advanced one pass small out, 167004
github, uncompressed literals optimal, advanced one pass small out, 156824 github, uncompressed literals optimal, advanced one pass small out, 156824
github, huffman literals, advanced one pass small out, 143457 github, huffman literals, advanced one pass small out, 143457
github, multithreaded with advanced params, advanced one pass small out, 167004
silesia, level -5, advanced streaming, 7152294 silesia, level -5, advanced streaming, 7152294
silesia, level -3, advanced streaming, 6789973 silesia, level -3, advanced streaming, 6789973
silesia, level -1, advanced streaming, 6191549 silesia, level -1, advanced streaming, 6191549
@ -368,6 +379,7 @@ silesia, explicit params, advanced str
silesia, uncompressed literals, advanced streaming, 5155424 silesia, uncompressed literals, advanced streaming, 5155424
silesia, uncompressed literals optimal, advanced streaming, 4325427 silesia, uncompressed literals optimal, advanced streaming, 4325427
silesia, huffman literals, advanced streaming, 5341357 silesia, huffman literals, advanced streaming, 5341357
silesia, multithreaded with advanced params, advanced streaming, 5155424
silesia.tar, level -5, advanced streaming, 7160440 silesia.tar, level -5, advanced streaming, 7160440
silesia.tar, level -3, advanced streaming, 6789026 silesia.tar, level -3, advanced streaming, 6789026
silesia.tar, level -1, advanced streaming, 6195465 silesia.tar, level -1, advanced streaming, 6195465
@ -393,6 +405,7 @@ silesia.tar, explicit params, advanced str
silesia.tar, uncompressed literals, advanced streaming, 5157995 silesia.tar, uncompressed literals, advanced streaming, 5157995
silesia.tar, uncompressed literals optimal, advanced streaming, 4321094 silesia.tar, uncompressed literals optimal, advanced streaming, 4321094
silesia.tar, huffman literals, advanced streaming, 5358083 silesia.tar, huffman literals, advanced streaming, 5358083
silesia.tar, multithreaded with advanced params, advanced streaming, 5158130
github, level -5, advanced streaming, 232744 github, level -5, advanced streaming, 232744
github, level -5 with dict, advanced streaming, 46718 github, level -5 with dict, advanced streaming, 46718
github, level -3, advanced streaming, 220611 github, level -3, advanced streaming, 220611
@ -432,6 +445,7 @@ github, explicit params, advanced str
github, uncompressed literals, advanced streaming, 167004 github, uncompressed literals, advanced streaming, 167004
github, uncompressed literals optimal, advanced streaming, 156824 github, uncompressed literals optimal, advanced streaming, 156824
github, huffman literals, advanced streaming, 143457 github, huffman literals, advanced streaming, 143457
github, multithreaded with advanced params, advanced streaming, 167004
silesia, level -5, old streaming, 7152294 silesia, level -5, old streaming, 7152294
silesia, level -3, old streaming, 6789973 silesia, level -3, old streaming, 6789973
silesia, level -1, old streaming, 6191549 silesia, level -1, old streaming, 6191549

1 Data Config Method Total compressed size
40 silesia uncompressed literals compress cctx 4862377
41 silesia uncompressed literals optimal compress cctx 4293262
42 silesia huffman literals compress cctx 6191548
43 silesia multithreaded with advanced params compress cctx 4862377
44 github level -5 compress cctx 232744
45 github level -5 with dict compress cctx 47294
46 github level -3 compress cctx 220611
79 github uncompressed literals compress cctx 136397
80 github uncompressed literals optimal compress cctx 133717
81 github huffman literals compress cctx 176575
82 github multithreaded with advanced params compress cctx 141473
83 silesia level -5 zstdcli 7152342
84 silesia level -3 zstdcli 6790021
85 silesia level -1 zstdcli 6191597
104 silesia uncompressed literals zstdcli 5155472
105 silesia uncompressed literals optimal zstdcli 4325475
106 silesia huffman literals zstdcli 5341405
107 silesia multithreaded with advanced params zstdcli compression error
108 silesia.tar level -5 zstdcli 7161160
109 silesia.tar level -3 zstdcli 6789865
110 silesia.tar level -1 zstdcli 6196433
130 silesia.tar uncompressed literals zstdcli 5158134
131 silesia.tar uncompressed literals optimal zstdcli 4321098
132 silesia.tar huffman literals zstdcli 5358479
133 silesia.tar multithreaded with advanced params zstdcli compression error
134 github level -5 zstdcli 234744
135 github level -5 with dict zstdcli 48718
136 github level -3 zstdcli 222611
169 github uncompressed literals zstdcli 169004
170 github uncompressed literals optimal zstdcli 158824
171 github huffman literals zstdcli 145457
172 github multithreaded with advanced params zstdcli compression error
173 silesia level -5 advanced one pass 7152294
174 silesia level -3 advanced one pass 6789969
175 silesia level -1 advanced one pass 6191548
195 silesia uncompressed literals advanced one pass 5155424
196 silesia uncompressed literals optimal advanced one pass 4325427
197 silesia huffman literals advanced one pass 5341356
198 silesia multithreaded with advanced params advanced one pass 5155424
199 silesia.tar level -5 advanced one pass 7160438
200 silesia.tar level -3 advanced one pass 6789024
201 silesia.tar level -1 advanced one pass 6195462
221 silesia.tar uncompressed literals advanced one pass 5157992
222 silesia.tar uncompressed literals optimal advanced one pass 4321094
223 silesia.tar huffman literals advanced one pass 5358079
224 silesia.tar multithreaded with advanced params advanced one pass 5158545
225 github level -5 advanced one pass 232744
226 github level -5 with dict advanced one pass 46718
227 github level -3 advanced one pass 220611
261 github uncompressed literals advanced one pass 167004
262 github uncompressed literals optimal advanced one pass 156824
263 github huffman literals advanced one pass 143457
264 github multithreaded with advanced params advanced one pass 167004
265 silesia level -5 advanced one pass small out 7152294
266 silesia level -3 advanced one pass small out 6789969
267 silesia level -1 advanced one pass small out 6191548
287 silesia uncompressed literals advanced one pass small out 5155424
288 silesia uncompressed literals optimal advanced one pass small out 4325427
289 silesia huffman literals advanced one pass small out 5341356
290 silesia multithreaded with advanced params advanced one pass small out 5155424
291 silesia.tar level -5 advanced one pass small out 7160438
292 silesia.tar level -3 advanced one pass small out 6789024
293 silesia.tar level -1 advanced one pass small out 6195462
313 silesia.tar uncompressed literals advanced one pass small out 5157992
314 silesia.tar uncompressed literals optimal advanced one pass small out 4321094
315 silesia.tar huffman literals advanced one pass small out 5358079
316 silesia.tar multithreaded with advanced params advanced one pass small out 5158545
317 github level -5 advanced one pass small out 232744
318 github level -5 with dict advanced one pass small out 46718
319 github level -3 advanced one pass small out 220611
353 github uncompressed literals advanced one pass small out 167004
354 github uncompressed literals optimal advanced one pass small out 156824
355 github huffman literals advanced one pass small out 143457
356 github multithreaded with advanced params advanced one pass small out 167004
357 silesia level -5 advanced streaming 7152294
358 silesia level -3 advanced streaming 6789973
359 silesia level -1 advanced streaming 6191549
379 silesia uncompressed literals advanced streaming 5155424
380 silesia uncompressed literals optimal advanced streaming 4325427
381 silesia huffman literals advanced streaming 5341357
382 silesia multithreaded with advanced params advanced streaming 5155424
383 silesia.tar level -5 advanced streaming 7160440
384 silesia.tar level -3 advanced streaming 6789026
385 silesia.tar level -1 advanced streaming 6195465
405 silesia.tar uncompressed literals advanced streaming 5157995
406 silesia.tar uncompressed literals optimal advanced streaming 4321094
407 silesia.tar huffman literals advanced streaming 5358083
408 silesia.tar multithreaded with advanced params advanced streaming 5158130
409 github level -5 advanced streaming 232744
410 github level -5 with dict advanced streaming 46718
411 github level -3 advanced streaming 220611
445 github uncompressed literals advanced streaming 167004
446 github uncompressed literals optimal advanced streaming 156824
447 github huffman literals advanced streaming 143457
448 github multithreaded with advanced params advanced streaming 167004
449 silesia level -5 old streaming 7152294
450 silesia level -3 old streaming 6789973
451 silesia level -1 old streaming 6191549