facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added support for multithreading parameters

Browse Source
This commit is contained in:
Yann Collet 2017-06-01 18:44:06 -07:00
parent c4a5a21c5c
commit c35e535002
4 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lib/compress/zstd_compress.c
View File

@ -157,6 +157,7 @@ struct ZSTD_CCtx_s {
U32 frameEnded;
/* Multi-threading */
U32 nbThreads;
ZSTDMT_CCtx* mtctx;
};
@ -357,6 +358,30 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->loadedDictEnd = 0;
return 0;
case ZSTD_p_nbThreads:
if (value==0) return 0;
#ifndef ZSTD_MULTITHREAD
if (value > 1) return ERROR(compressionParameter_unsupported);
#endif
if ((value>1) && (cctx->nbThreads != value)) {
ZSTDMT_freeCCtx(cctx->mtctx);
cctx->nbThreads = value;
cctx->mtctx = ZSTDMT_createCCtx(value);
if (cctx->mtctx == NULL) return ERROR(memory_allocation);
}
cctx->nbThreads = 1;
return 0;
case ZSTDMT_p_jobSize:
if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
case ZSTDMT_p_overlapSizeLog:
if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
case ZSTD_p_rawContentDict : /* load dictionary in "content-only" mode (no header analysis) (default:0) */
cctx->forceRawDict = value>0;
return 0;

2
lib/compress/zstdmt_compress.h
View File

@ -33,7 +33,7 @@ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
unsigned nbThreads);
unsigned nbThreads); /* not ready yet */
/* === Simple buffer-to-butter one-pass function === */

13
lib/zstd.h
View File

@ -491,7 +491,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* These functions make it possible to estimate memory usage
* of a future target object, before its allocation,
* given a set of parameters, which vary depending on target object.
* The objective is to guide decision before allocation. */
* The objective is to guide decision before allocation.
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
@ -695,17 +696,17 @@ typedef enum {
ZSTD_p_rawContentDict, /* load dictionary in "content-only" mode (no header analysis) (default:0) */
/* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
#if 0
/* multi-threading parameters (not ready yet !) */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
* More threads improve speed, but increases also memory usage */
ZSTDMT_p_jobSize, /* Size of a compression job. Each job is compressed in parallel.
* More threads improve speed, but also increase memory usage.
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
* Special: value 0 means "do not change nbThreads" */
ZSTDMT_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
#endif
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,