From 110cc14bab4e4dd689201a6334d9b992e7e1fc69 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 19 Nov 2015 12:02:28 +0100 Subject: [PATCH] fixed bug #73 notified by @nemequ --- lib/zstd_compress.c | 7 ++++--- programs/fuzzer.c | 23 ++++++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 53dfc742..ccfabf89 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1218,8 +1218,8 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( /* common lazy function, to be inlined */ FORCE_INLINE size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, const void* src, size_t srcSize, - const U32 searchMethod, const U32 deep) /* 0 : hc; 1 : bt */ + void* dst, size_t maxDstSize, const void* src, size_t srcSize, + const U32 searchMethod, const U32 deep) /* searchMethod : 0 = hc; 1 = bt */ { seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const istart = (const BYTE*)src; @@ -1496,6 +1496,7 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(ctx->params.strategy, ctx->lowLimit < ctx->dictLimit); + if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */ return blockCompressor(ctx, dst, maxDstSize, src, srcSize); } @@ -1536,7 +1537,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, op[0] = (BYTE)(cSize>>16); op[1] = (BYTE)(cSize>>8); op[2] = (BYTE)cSize; - op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */ + op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */ cSize += 3; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index ec1ccbd5..d3af3ff8 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -63,7 +63,7 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const U32 nbTestsDefault = 32 KB; +static const U32 nbTestsDefault = 30000; #define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; @@ -89,6 +89,8 @@ static U32 g_time = 0; /********************************************************* * Fuzzer functions *********************************************************/ +#define MAX(a,b) ((a)>(b)?(a):(b)) + static U32 FUZ_GetMilliStart(void) { struct timeb tb; @@ -299,6 +301,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 sampleSizeLog, buffNb, cLevelMod; U64 crcOrig, crcDest; int cLevel; + BYTE* sampleBuffer; /* init */ DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); @@ -325,13 +328,17 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); - crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0); - /* HC compression test */ -#define MAX(a,b) ((a)>(b)?(a):(b)) + /* create sample buffer (to catch read error with valgrind & sanitizers) */ + sampleBuffer = (BYTE*)malloc(sampleSize); + CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); + memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); + crcOrig = XXH64(sampleBuffer, sampleSize, 0); + + /* compression test */ cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; - cSize = ZSTD_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel); + cSize = ZSTD_compressCCtx(hcctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ @@ -343,7 +350,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit static const U32 endMark = 0x4DC2B1A9; U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); - errorCode = ZSTD_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel); + errorCode = ZSTD_compressCCtx(hcctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); @@ -354,7 +361,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); - CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize); + CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); + + free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ {