facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2611 from facebook/smallerJobs 

allow jobSize to be as low as 512 KB
dev
Yann Collet 2021-05-05 00:03:29 -07:00 committed by GitHub
parent 8389a5122b 1026b9fa10
commit c077f257b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/compress/zstdmt_compress.c
View File

@ -1240,9 +1240,8 @@ size_t ZSTDMT_initCStream_internal(
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20;
assert(jobSizeMB >= 1);
U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10);
U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;

6
lib/compress/zstdmt_compress.h
View File

@ -32,11 +32,11 @@
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX
#ifndef ZSTDMT_NBWORKERS_MAX /* can be modified at compile time */
# define ZSTDMT_NBWORKERS_MAX 200
#endif
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#ifndef ZSTDMT_JOBSIZE_MIN /* can be modified at compile time */
# define ZSTDMT_JOBSIZE_MIN (512 KB)
#endif
#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30)
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))

8
tests/fuzzer.c
View File

@ -1589,6 +1589,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
{ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
int const jobSize = 512 KB;
int value;
/* Check that the overlap log and job size are unset. */
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
@ -1597,19 +1598,18 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_EQ(value, 0);
/* Set and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB);
CHECK_EQ(value, jobSize);
/* Set the number of workers and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB);
CHECK_EQ(value, jobSize);
ZSTD_freeCCtxParams(params);
}
DISPLAYLEVEL(3, "OK \n");