From 997f9ee2efebc347ffceefb5ea34c53ee992e143 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 21 Aug 2015 02:44:20 +0100 Subject: [PATCH] Added : fuzzer tests : decompressing noisy src --- lib/fse.c | 35 ++++--- lib/zstd.c | 230 +++++++++------------------------------------- programs/fuzzer.c | 48 +++++++++- 3 files changed, 109 insertions(+), 204 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index b1a36574..a577e81e 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -703,7 +703,7 @@ static short FSE_abs(short a) ****************************************************************/ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) { - size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1 + 1; /* last +1 : written by U16 */ + size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ } @@ -804,15 +804,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, } -size_t FSE_writeNCount (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */ if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */ - if (headerBufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); } @@ -913,9 +913,9 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } else { - ip = iend - 4; bitCount -= (int)(8 * (iend - 4 - ip)); - } + ip = iend - 4; + } bitStream = FSE_readLE32(ip) >> (bitCount & 31); } } @@ -967,9 +967,12 @@ void FSE_freeCTable (FSE_CTable* ct) unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) { U32 tableLog = maxTableLog; + U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2; /* Accuracy can be reduced */ - if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2; /* Need a minimum to safely represent all symbol values */ + if (minBitsSrc < tableLog + 3) tableLog = minBitsSrc-3; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; return tableLog; @@ -1076,7 +1079,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */ if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */ - if ((1U<= unsigned FSE_reloadDStream(FSE_DStream_t* bitD) { + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return FSE_DStream_tooFar; + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { bitD->ptr -= bitD->bitsConsumed >> 3; @@ -1565,20 +1571,19 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD) if (bitD->ptr == bitD->start) { if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer; - if (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8) return FSE_DStream_completed; - return FSE_DStream_tooFar; + return FSE_DStream_completed; } { U32 nbBytes = bitD->bitsConsumed >> 3; U32 result = FSE_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* note : necessarily ptr > start */ + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ result = FSE_DStream_endOfBuffer; } bitD->ptr -= nbBytes; bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = FSE_readLEST(bitD->ptr); /* note : necessarily srcSize > sizeof(bitD) */ + bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ return result; } } @@ -2044,7 +2049,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size FSE_CStream_t bitC; /* init */ - if (dstSize < 8) return 0; /* need a minimum for jumpTable and first symbols */ + if (dstSize < 8) return 0; op += 6; /* jump Table -- could be optimized by delta / deviation */ errorCode = FSE_initCStream(&bitC, op, oend-op); if (FSE_isError(errorCode)) return 0; diff --git a/lib/zstd.c b/lib/zstd.c index ad03e553..5414a56c 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1245,6 +1245,9 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, U32 LLlog, Offlog, MLlog; size_t dumpsLength; + /* check */ + if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize; + /* SeqHead */ *nbSeq = ZSTD_readLE16(ip); ip+=2; LLtype = *ip >> 6; @@ -1265,6 +1268,9 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, *dumpsPtr = ip; ip += dumpsLength; + /* check */ + if (ip > iend-1) return (size_t)-ZSTD_ERROR_SrcSize; + /* sequences */ { S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL and MaxOff */ @@ -1284,6 +1290,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxLL; headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableLL, norm, max, LLlog); } @@ -1301,6 +1308,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxOff; headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableOffb, norm, max, Offlog); } @@ -1318,6 +1326,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxML; headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableML, norm, max, MLlog); } @@ -1400,7 +1409,10 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } -static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, BYTE* const oend) +static size_t ZSTD_execSequence(BYTE* op, + seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + BYTE* const base, BYTE* const oend) { static const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ @@ -1411,6 +1423,7 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B /* check */ if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; + if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption; /* copy Literals */ if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8)) @@ -1430,6 +1443,10 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B size_t qutt = 12; U64 saved[2]; + /* check */ + if (match < base) return (size_t)-ZSTD_ERROR_corruption; + if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption; + /* save beginning of literal sequence, in case of write overlap */ if (overlapRisk) { @@ -1470,6 +1487,18 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B return endMatch-ostart; } +typedef struct ZSTD_Dctx_s +{ + U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; + U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; + void* previousDstEnd; + void* base; + size_t expected; + blockType_t bType; + U32 phase; +} dctx_t; + static size_t ZSTD_decompressSequences( void* ctx, @@ -1477,6 +1506,7 @@ static size_t ZSTD_decompressSequences( const void* seqStart, size_t seqSize, const BYTE* litStart, size_t litSize) { + dctx_t* dctx = (dctx_t*)ctx; const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE* const)dst; @@ -1487,9 +1517,10 @@ static size_t ZSTD_decompressSequences( const BYTE* const litEnd = litStart + litSize; int nbSeq; const BYTE* dumps; - FSE_DTable* DTableML = (FSE_DTable*)ctx; - FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog); - FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog); + U32* DTableLL = dctx->LLTable; + U32* DTableML = dctx->MLTable; + U32* DTableOffb = dctx->OffTable; + BYTE* const base = (BYTE*) (dctx->base); /* Build Decoding Tables */ errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, @@ -1515,7 +1546,7 @@ static size_t ZSTD_decompressSequences( size_t oneSeqSize; nbSeq--; ZSTD_decodeSequence(&sequence, &seqState); - oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, oend); + oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } @@ -1558,175 +1589,6 @@ static size_t ZSTD_decompressBlock( } -#if 0 -FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* op = ostart; - BYTE* const oend = ostart + maxDstSize; - size_t errorCode; - size_t lastLLSize; - const BYTE* dumps; - const BYTE* litPtr; - const BYTE* litEnd; - const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */ - const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ - FSE_DTable* DTableML = (FSE_DTable*)ctx; - FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog); - FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog); - - /* blockType == blockCompressed, srcSize is trusted */ - - /* Decode literals sub-block */ - errorCode = ZSTD_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, src, srcSize); - if (ZSTD_isError(errorCode)) return errorCode; - ip += errorCode; - - /* Build Decoding Tables */ - errorCode = ZSTD_decodeSeqHeaders(&lastLLSize, &dumps, - DTableLL, DTableML, DTableOffb, - ip, iend-ip); - if (ZSTD_isError(errorCode)) return errorCode; - /* end pos */ - if ((litPtr>=ostart) && (litPtr<=oend)) /* decoded literals are into dst buffer */ - litEnd = oend - lastLLSize; - else - litEnd = ip - lastLLSize; - ip += errorCode; - - /* LZ Sequences */ - { - FSE_DStream_t DStream; - FSE_DState_t stateLL, stateOffb, stateML; - size_t prevOffset = 0, offset = 0; - - FSE_initDStream(&DStream, ip, iend-ip); - FSE_initDState(&stateLL, &DStream, DTableLL); - FSE_initDState(&stateOffb, &DStream, DTableOffb); - FSE_initDState(&stateML, &DStream, DTableML); - - while (FSE_reloadDStream(&DStream)<2) - { - U32 nbBits, offsetCode; - const BYTE* match; - size_t litLength; - size_t matchLength; - size_t newOffset; - -_another_round: - - /* Literals */ - litLength = FSE_decodeSymbol(&stateLL, &DStream); - if (litLength) prevOffset = offset; - if (litLength == MaxLL) - { - BYTE add = *dumps++; - if (add < 255) litLength += add; - else - { - litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; - dumps += 3; - } - } - if (((size_t)(litPtr - op) < 8) || ((size_t)(oend-(litPtr+litLength)) < 8)) - memmove(op, litPtr, litLength); /* overwrite risk */ - else - ZSTD_wildcopy(op, litPtr, litLength); - op += litLength; - litPtr += litLength; - - /* Offset */ - offsetCode = FSE_decodeSymbol(&stateOffb, &DStream); - if (ZSTD_32bits()) FSE_reloadDStream(&DStream); - nbBits = offsetCode - 1; - if (offsetCode==0) nbBits = 0; /* cmove */ - newOffset = FSE_readBits(&DStream, nbBits); - if (ZSTD_32bits()) FSE_reloadDStream(&DStream); - newOffset += (size_t)1 << nbBits; - if (offsetCode==0) newOffset = prevOffset; - match = op - newOffset; - prevOffset = offset; - offset = newOffset; - - /* MatchLength */ - matchLength = FSE_decodeSymbol(&stateML, &DStream); - if (matchLength == MaxML) - { - BYTE add = *dumps++; - if (add < 255) matchLength += add; - else - { - matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */ - dumps += 3; - } - } - matchLength += MINMATCH; - - /* copy Match */ - { - BYTE* const endMatch = op + matchLength; - size_t qutt=12; - U64 saved[2]; - const U32 overlapRisk = (((size_t)(litPtr - endMatch)) < 12); - - /* save beginning of literal sequence, in case of write overlap */ - if (overlapRisk) - { - if ((endMatch + qutt) > oend) qutt = oend-endMatch; - memcpy(saved, endMatch, qutt); - } - - if (offset < 8) - { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - ZSTD_copy4(op+4, match); - match -= dec64; - } else { ZSTD_copy8(op, match); } - - if (endMatch > oend-12) - { - if (op < oend-16) - { - ZSTD_wildcopy(op+8, match+8, (oend-8) - (op+8)); - match += (oend-8) - op; - op = oend-8; - } - while (op 2) return (size_t)-ZSTD_ERROR_GENERIC; /* requested too much : data is corrupted */ - if (!FSE_endOfDState(&stateLL) && !FSE_endOfDState(&stateML) && !FSE_endOfDState(&stateOffb)) goto _another_round; /* some ultra-compressible sequence remain ! */ - if (litPtr != litEnd) goto _another_round; /* literals not entirely spent */ - - /* last literal segment */ - if (op != litPtr) memmove(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op-ostart; -} -#endif - - static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; @@ -1784,11 +1646,11 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const return op-ostart; } - size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)]; - return ZSTD_decompressDCtx(ctx, dst, maxDstSize, src, srcSize); + dctx_t ctx; + ctx.base = dst; + return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } @@ -1796,21 +1658,14 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src * Streaming Decompression API *******************************/ -typedef struct ZSTD_Dctx_s -{ - U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)]; - size_t expected; - blockType_t bType; - U32 phase; -} dctx_t; - - ZSTD_Dctx* ZSTD_createDCtx(void) { ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx)); if (dctx==NULL) return NULL; dctx->expected = ZSTD_frameHeaderSize; dctx->phase = 0; + dctx->previousDstEnd = NULL; + dctx->base = NULL; return dctx; } @@ -1832,6 +1687,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co /* Sanity check */ if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize; + if (dst != ctx->previousDstEnd) /* not contiguous */ + ctx->base = dst; /* Decompress : frame header */ if (ctx->phase == 0) @@ -1887,6 +1744,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co } ctx->phase = 1; ctx->expected = ZSTD_blockHeaderSize; + ctx->previousDstEnd = (void*)( ((char*)dst) + rSize); return rSize; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 6416536f..35aee2df 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -177,8 +177,7 @@ static void FUZ_generateSynthetic(void* buffer, size_t bufferSize, double proba, } -/* -static unsigned FUZ_highbit(U32 v32) +static unsigned FUZ_highbit32(U32 v32) { unsigned nbBits = 0; if (v32==0) return 0; @@ -189,7 +188,6 @@ static unsigned FUZ_highbit(U32 v32) } return nbBits; } -*/ static int basicUnitTests(U32 seed, double compressibility) @@ -405,6 +403,50 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } + + /* noisy src decompression test */ + if (cSize > 6) + { + const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); + size_t pos = 4; /* preserve magic number (too easy to detect) */ + U32 nbBits = FUZ_rand(&lseed) % maxNbBits; + size_t mask = (1<0) nbBits--; + mask = (1<sampleSize), + "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); + memcpy(&endCheck, dstBuffer+sampleSize, 4); + CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); + } + } } DISPLAY("\rAll fuzzer tests completed \n");