facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2538 from senhuang42/monotonicity_test

Browse Source 
Add memory monotonicity test over srcSize
This commit is contained in:
sen 2021-03-22 16:54:34 -04:00 committed by GitHub
commit c48889f097
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 52 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/compress/zstd_compress.c
View File

@ -1390,8 +1390,15 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
{
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
return ZSTD_estimateCCtxSize_usingCParams(cParams);
int tier = 0;
size_t largestSize = 0;
static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
for (; tier < 4; ++tier) {
/* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
}
return largestSize;
}
size_t ZSTD_estimateCCtxSize(int compressionLevel)
@ -1399,6 +1406,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
int level;
size_t memBudget = 0;
for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
/* Ensure monotonically increasing memory usage as compression level increases */
size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
if (newMB > memBudget) memBudget = newMB;
}

42
tests/fuzzer.c
View File

@ -3181,6 +3181,48 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
{
int level = 1;
size_t prevSize = 0;
for (; level < ZSTD_maxCLevel(); ++level) {
size_t const currSize = ZSTD_estimateCCtxSize(level);
if (prevSize > currSize) {
DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
prevSize, level-1, currSize, level);
goto _output_error;
}
prevSize = currSize;
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
{
size_t const kSizeIncrement = 2 KB;
int level = -3;
for (; level <= ZSTD_maxCLevel(); ++level) {
size_t dictSize = 0;
for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
size_t srcSize = 2 KB;
for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
if (cctxSizeUsingLevel < cctxSizeUsingCParams
|| ZSTD_isError(cctxSizeUsingCParams)
|| ZSTD_isError(cctxSizeUsingLevel)) {
DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
goto _output_error;
}
}
}
}
}
DISPLAYLEVEL(3, "OK \n");
#endif
_end: