facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2611 from facebook/smallerJobs 

allow jobSize to be as low as 512 KB
dev
Yann Collet 2021-05-05 00:03:29 -07:00 committed by GitHub
parent 8389a5122b 1026b9fa10
commit c077f257b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/compress/zstdmt_compress.c
View File

@ -1240,9 +1240,8 @@ size_t ZSTDMT_initCStream_internal(
if (params.rsyncable) { if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */ /* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10);
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20; U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10);
assert(jobSizeMB >= 1);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits); DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0; mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;

6
lib/compress/zstdmt_compress.h
View File

@ -32,11 +32,11 @@
/* === Constants === */ /* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX #ifndef ZSTDMT_NBWORKERS_MAX /* can be modified at compile time */
# define ZSTDMT_NBWORKERS_MAX 200 # define ZSTDMT_NBWORKERS_MAX 200
#endif #endif
#ifndef ZSTDMT_JOBSIZE_MIN #ifndef ZSTDMT_JOBSIZE_MIN /* can be modified at compile time */
# define ZSTDMT_JOBSIZE_MIN (1 MB) # define ZSTDMT_JOBSIZE_MIN (512 KB)
#endif #endif
#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) #define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30)
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))

8
tests/fuzzer.c
View File

@ -1589,6 +1589,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
{ ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
int const jobSize = 512 KB;
int value; int value;
/* Check that the overlap log and job size are unset. */ /* Check that the overlap log and job size are unset. */
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
@ -1597,19 +1598,18 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_EQ(value, 0); CHECK_EQ(value, 0);
/* Set and check the overlap log and job size. */ /* Set and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) ); CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5); CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB); CHECK_EQ(value, jobSize);
/* Set the number of workers and check the overlap log and job size. */ /* Set the number of workers and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5); CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB); CHECK_EQ(value, jobSize);
ZSTD_freeCCtxParams(params); ZSTD_freeCCtxParams(params);
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");