facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1928 from bimbashrestha/bimba-oss 

[fuzz] Adding bool to check if there is enough room to emit noCompress Superblocks
dev
Yann Collet 2019-12-19 12:58:24 -08:00 committed by GitHub
parent 07ad866754 56415efc76
commit 1c9ac4ccf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 58 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/compress/zstd_compress.c
View File

@ -2562,6 +2562,14 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
/* This bool is set if there is enough room to output all noCompress superblocks.
* Just checks if the number of compressed blocks we can fit in dstCapacity is
* greater than the optimistic number of blocks we still have remaining.
* This might be UNset when data is uncompressable and we're streaming. */
const int enoughDstCapacityForNoCompressSuperBlocks =
(dstCapacity / (blockSize + 7 /* header + checksum */)) > (srcSize / blockSize);
assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
@ -2586,7 +2594,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
{ size_t cSize;
int useTargetCBlockSize = ZSTD_useTargetCBlockSize(&cctx->appliedParams);
if (useTargetCBlockSize) {
if (useTargetCBlockSize && enoughDstCapacityForNoCompressSuperBlocks) {
cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize);
} else {

50
tests/fuzzer.c
View File

@ -489,6 +489,54 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++)
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
/* This is the number of bytes to stream before ending. This value
* was obtained by trial and error :/. */
const size_t streamCompressThreshold = 161792;
const size_t streamCompressDelta = 1024;
/* The first 1/3 of the buffer is compressible and the last 2/3 is
* uncompressible. This is an approximation of the type of data
* the fuzzer generated to catch this bug. Streams like this were making
* zstd generate noCompress superblocks (which are larger than the src
* they come from). Do this enough times, and we'll run out of room
* and throw a dstSize_tooSmall error. */
const size_t compressiblePartSize = srcSize/3;
const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
/* Setting target block size so that superblock is used */
assert(cctx != NULL);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
{ size_t read;
for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
ZSTD_inBuffer in = {src, streamCompressDelta, 0};
ZSTD_outBuffer out = {dst, dstCapacity, 0};
assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)));
assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)));
src += streamCompressDelta; srcSize -= streamCompressDelta;
dst += out.pos; dstCapacity -= out.pos;}}
/* This is trying to catch a dstSize_tooSmall error */
{ ZSTD_inBuffer in = {src, srcSize, 0};
ZSTD_outBuffer out = {dst, dstCapacity, 0};
assert(!ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)));}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
{
/* This tests whether or not we leave enough room for the checksum at the end
@ -501,7 +549,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
{ int level = -1;
ZSTD_CCtx* cctx = ZSTD_createCCtx();