Added : fuzzer tests : decompressing noisy src

This commit is contained in:
Yann Collet 2015-08-21 02:44:20 +01:00
parent 7d20acd340
commit 997f9ee2ef
3 changed files with 109 additions and 204 deletions

View File

@ -703,7 +703,7 @@ static short FSE_abs(short a)
****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 1 + 1; /* last +1 : written by U16 */
size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
@ -804,15 +804,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
}
size_t FSE_writeNCount (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported */
if (headerBufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
return FSE_writeNCount_generic(header, headerBufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
@ -913,9 +913,9 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
}
else
{
ip = iend - 4;
bitCount -= (int)(8 * (iend - 4 - ip));
}
ip = iend - 4;
}
bitStream = FSE_readLE32(ip) >> (bitCount & 31);
}
}
@ -967,9 +967,12 @@ void FSE_freeCTable (FSE_CTable* ct)
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
U32 tableLog = maxTableLog;
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2; /* Accuracy can be reduced */
if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2; /* Need a minimum to safely represent all symbol values */
if (minBitsSrc < tableLog + 3) tableLog = minBitsSrc-3; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
@ -1076,7 +1079,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog < FSE_MIN_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_GENERIC; /* Unsupported size */
if ((1U<<tableLog) <= maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
//if ((1U<<tableLog) <= maxSymbolValue) return (size_t)-FSE_ERROR_GENERIC; /* Too small tableLog, compression potentially impossible */
{
U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
@ -1555,6 +1558,9 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >=
unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
return FSE_DStream_tooFar;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
{
bitD->ptr -= bitD->bitsConsumed >> 3;
@ -1565,20 +1571,19 @@ unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer;
if (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8) return FSE_DStream_completed;
return FSE_DStream_tooFar;
return FSE_DStream_completed;
}
{
U32 nbBytes = bitD->bitsConsumed >> 3;
U32 result = FSE_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (U32)(bitD->ptr - bitD->start); /* note : necessarily ptr > start */
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
result = FSE_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
bitD->bitContainer = FSE_readLEST(bitD->ptr); /* note : necessarily srcSize > sizeof(bitD) */
bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
return result;
}
}
@ -2044,7 +2049,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
FSE_CStream_t bitC;
/* init */
if (dstSize < 8) return 0; /* need a minimum for jumpTable and first symbols */
if (dstSize < 8) return 0;
op += 6; /* jump Table -- could be optimized by delta / deviation */
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;

View File

@ -1245,6 +1245,9 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
U32 LLlog, Offlog, MLlog;
size_t dumpsLength;
/* check */
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
/* SeqHead */
*nbSeq = ZSTD_readLE16(ip); ip+=2;
LLtype = *ip >> 6;
@ -1265,6 +1268,9 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
*dumpsPtr = ip;
ip += dumpsLength;
/* check */
if (ip > iend-1) return (size_t)-ZSTD_ERROR_SrcSize;
/* sequences */
{
S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL and MaxOff */
@ -1284,6 +1290,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableLL, norm, max, LLlog);
}
@ -1301,6 +1308,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableOffb, norm, max, Offlog);
}
@ -1318,6 +1326,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog);
}
@ -1400,7 +1409,10 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
}
static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, BYTE* const oend)
static size_t ZSTD_execSequence(BYTE* op,
seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
BYTE* const base, BYTE* const oend)
{
static const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
@ -1411,6 +1423,7 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B
/* check */
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
/* copy Literals */
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
@ -1430,6 +1443,10 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B
size_t qutt = 12;
U64 saved[2];
/* check */
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
/* save beginning of literal sequence, in case of write overlap */
if (overlapRisk)
{
@ -1470,6 +1487,18 @@ static size_t ZSTD_execSequence(BYTE* op, seq_t sequence, const BYTE** litPtr, B
return endMatch-ostart;
}
typedef struct ZSTD_Dctx_s
{
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
void* previousDstEnd;
void* base;
size_t expected;
blockType_t bType;
U32 phase;
} dctx_t;
static size_t ZSTD_decompressSequences(
void* ctx,
@ -1477,6 +1506,7 @@ static size_t ZSTD_decompressSequences(
const void* seqStart, size_t seqSize,
const BYTE* litStart, size_t litSize)
{
dctx_t* dctx = (dctx_t*)ctx;
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst;
@ -1487,9 +1517,10 @@ static size_t ZSTD_decompressSequences(
const BYTE* const litEnd = litStart + litSize;
int nbSeq;
const BYTE* dumps;
FSE_DTable* DTableML = (FSE_DTable*)ctx;
FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog);
FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog);
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
BYTE* const base = (BYTE*) (dctx->base);
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
@ -1515,7 +1546,7 @@ static size_t ZSTD_decompressSequences(
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, oend);
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
@ -1558,175 +1589,6 @@ static size_t ZSTD_decompressBlock(
}
#if 0
FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
size_t errorCode;
size_t lastLLSize;
const BYTE* dumps;
const BYTE* litPtr;
const BYTE* litEnd;
const int dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; /* added */
const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
FSE_DTable* DTableML = (FSE_DTable*)ctx;
FSE_DTable* DTableLL = DTableML + FSE_DTABLE_SIZE_U32(MLFSELog);
FSE_DTable* DTableOffb = DTableLL + FSE_DTABLE_SIZE_U32(LLFSELog);
/* blockType == blockCompressed, srcSize is trusted */
/* Decode literals sub-block */
errorCode = ZSTD_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, src, srcSize);
if (ZSTD_isError(errorCode)) return errorCode;
ip += errorCode;
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&lastLLSize, &dumps,
DTableLL, DTableML, DTableOffb,
ip, iend-ip);
if (ZSTD_isError(errorCode)) return errorCode;
/* end pos */
if ((litPtr>=ostart) && (litPtr<=oend)) /* decoded literals are into dst buffer */
litEnd = oend - lastLLSize;
else
litEnd = ip - lastLLSize;
ip += errorCode;
/* LZ Sequences */
{
FSE_DStream_t DStream;
FSE_DState_t stateLL, stateOffb, stateML;
size_t prevOffset = 0, offset = 0;
FSE_initDStream(&DStream, ip, iend-ip);
FSE_initDState(&stateLL, &DStream, DTableLL);
FSE_initDState(&stateOffb, &DStream, DTableOffb);
FSE_initDState(&stateML, &DStream, DTableML);
while (FSE_reloadDStream(&DStream)<2)
{
U32 nbBits, offsetCode;
const BYTE* match;
size_t litLength;
size_t matchLength;
size_t newOffset;
_another_round:
/* Literals */
litLength = FSE_decodeSymbol(&stateLL, &DStream);
if (litLength) prevOffset = offset;
if (litLength == MaxLL)
{
BYTE add = *dumps++;
if (add < 255) litLength += add;
else
{
litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;
dumps += 3;
}
}
if (((size_t)(litPtr - op) < 8) || ((size_t)(oend-(litPtr+litLength)) < 8))
memmove(op, litPtr, litLength); /* overwrite risk */
else
ZSTD_wildcopy(op, litPtr, litLength);
op += litLength;
litPtr += litLength;
/* Offset */
offsetCode = FSE_decodeSymbol(&stateOffb, &DStream);
if (ZSTD_32bits()) FSE_reloadDStream(&DStream);
nbBits = offsetCode - 1;
if (offsetCode==0) nbBits = 0; /* cmove */
newOffset = FSE_readBits(&DStream, nbBits);
if (ZSTD_32bits()) FSE_reloadDStream(&DStream);
newOffset += (size_t)1 << nbBits;
if (offsetCode==0) newOffset = prevOffset;
match = op - newOffset;
prevOffset = offset;
offset = newOffset;
/* MatchLength */
matchLength = FSE_decodeSymbol(&stateML, &DStream);
if (matchLength == MaxML)
{
BYTE add = *dumps++;
if (add < 255) matchLength += add;
else
{
matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3;
}
}
matchLength += MINMATCH;
/* copy Match */
{
BYTE* const endMatch = op + matchLength;
size_t qutt=12;
U64 saved[2];
const U32 overlapRisk = (((size_t)(litPtr - endMatch)) < 12);
/* save beginning of literal sequence, in case of write overlap */
if (overlapRisk)
{
if ((endMatch + qutt) > oend) qutt = oend-endMatch;
memcpy(saved, endMatch, qutt);
}
if (offset < 8)
{
const int dec64 = dec64table[offset];
op[0] = match[0];
op[1] = match[1];
op[2] = match[2];
op[3] = match[3];
match += dec32table[offset];
ZSTD_copy4(op+4, match);
match -= dec64;
} else { ZSTD_copy8(op, match); }
if (endMatch > oend-12)
{
if (op < oend-16)
{
ZSTD_wildcopy(op+8, match+8, (oend-8) - (op+8));
match += (oend-8) - op;
op = oend-8;
}
while (op<endMatch) *op++ = *match++;
}
else
ZSTD_wildcopy(op+8, match+8, matchLength-8); /* works even if matchLength < 8 */
op = endMatch;
/* restore, in case of overlap */
if (overlapRisk)
memcpy(endMatch, saved, qutt);
}
}
/* check if reached exact end */
if (FSE_reloadDStream(&DStream) > 2) return (size_t)-ZSTD_ERROR_GENERIC; /* requested too much : data is corrupted */
if (!FSE_endOfDState(&stateLL) && !FSE_endOfDState(&stateML) && !FSE_endOfDState(&stateOffb)) goto _another_round; /* some ultra-compressible sequence remain ! */
if (litPtr != litEnd) goto _another_round; /* literals not entirely spent */
/* last literal segment */
if (op != litPtr) memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
return op-ostart;
}
#endif
static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
@ -1784,11 +1646,11 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
return op-ostart;
}
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
return ZSTD_decompressDCtx(ctx, dst, maxDstSize, src, srcSize);
dctx_t ctx;
ctx.base = dst;
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
@ -1796,21 +1658,14 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
* Streaming Decompression API
*******************************/
typedef struct ZSTD_Dctx_s
{
U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
size_t expected;
blockType_t bType;
U32 phase;
} dctx_t;
ZSTD_Dctx* ZSTD_createDCtx(void)
{
ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
if (dctx==NULL) return NULL;
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
return dctx;
}
@ -1832,6 +1687,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
/* Sanity check */
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
if (dst != ctx->previousDstEnd) /* not contiguous */
ctx->base = dst;
/* Decompress : frame header */
if (ctx->phase == 0)
@ -1887,6 +1744,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
}
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
return rSize;
}

View File

@ -177,8 +177,7 @@ static void FUZ_generateSynthetic(void* buffer, size_t bufferSize, double proba,
}
/*
static unsigned FUZ_highbit(U32 v32)
static unsigned FUZ_highbit32(U32 v32)
{
unsigned nbBits = 0;
if (v32==0) return 0;
@ -189,7 +188,6 @@ static unsigned FUZ_highbit(U32 v32)
}
return nbBits;
}
*/
static int basicUnitTests(U32 seed, double compressibility)
@ -405,6 +403,50 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
/* noisy src decompression test */
if (cSize > 6)
{
const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
size_t pos = 4; /* preserve magic number (too easy to detect) */
U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
size_t mask = (1<<nbBits) - 1;
size_t skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
while (pos < cSize)
{
/* add noise */
size_t noiseStart, noiseLength;
nbBits = FUZ_rand(&lseed) % maxNbBits;
if (nbBits>0) nbBits--;
mask = (1<<nbBits) - 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
/* keep some original src */
nbBits = FUZ_rand(&lseed) % maxNbBits;
mask = (1<<nbBits) - 1;
skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
}
/* decompress noisy source */
{
const U32 endMark = 0xA9B1C3D6;
U32 endCheck;
size_t errorCode;
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
}
}
}
DISPLAY("\rAll fuzzer tests completed \n");