From fee8e240c738d9e1b9a63401b5b90ae7e516fde7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 24 Aug 2015 15:47:04 +0100 Subject: [PATCH 1/3] Fixed decoding error #11 (reported by @magv) --- lib/zstd.c | 4 +++- programs/fileio.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index bcc16c87..b93cca4e 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1269,7 +1269,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, ip += dumpsLength; /* check */ - if (ip > iend-1) return (size_t)-ZSTD_ERROR_SrcSize; + if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ /* sequences */ { @@ -1300,6 +1300,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, U32 max; case bt_rle : Offlog = 0; + if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */ FSE_buildDTable_rle(DTableOffb, *ip++); break; case bt_raw : Offlog = Offbits; @@ -1318,6 +1319,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, U32 max; case bt_rle : MLlog = 0; + if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */ FSE_buildDTable_rle(DTableML, *ip++); break; case bt_raw : MLlog = MLbits; diff --git a/programs/fileio.c b/programs/fileio.c index ee23dc01..c137c78b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -360,12 +360,13 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* Decode block */ decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); + if (ZSTD_isError(decodedSize)) EXM_THROW(35, "Decoding error : input corrupted"); if (decodedSize) /* not a header */ { /* Write block */ sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); + if (sizeCheck != decodedSize) EXM_THROW(36, "Write error : unable to write data block to destination file"); filesize += decodedSize; op += decodedSize; if (op==oend) op = outBuff; From 1885029ba15af286c07e3d54bfcd253b97d7974c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 24 Aug 2015 20:17:11 +0100 Subject: [PATCH 2/3] Fixed a few issues found by AFL (American Fuzzy Lop) --- lib/zstd.c | 7 ++++++- programs/datagencli.c | 2 +- programs/fileio.c | 8 +++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index b93cca4e..462b335a 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1172,7 +1172,12 @@ static size_t ZSTD_decompressLiterals(void* ctx, BYTE* const oend = op + maxDstSize; const BYTE* ip = (const BYTE*)src; size_t errorCode; - size_t litSize = ip[1] + (ip[0]<<8); + size_t litSize; + + /* check : minimum 2, for litSize, +1, for content */ + if (srcSize <= 3) return (size_t)-ZSTD_ERROR_corruption; + + litSize = ip[1] + (ip[0]<<8); litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh.... op = oend - litSize; diff --git a/programs/datagencli.c b/programs/datagencli.c index 2665c54b..ce581baf 100644 --- a/programs/datagencli.c +++ b/programs/datagencli.c @@ -81,7 +81,7 @@ static int usage(char* programName) { DISPLAY( "Compressible data generator\n"); DISPLAY( "Usage :\n"); - DISPLAY( " %s [size] [args]\n", programName); + DISPLAY( " %s [args]\n", programName); DISPLAY( "\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); diff --git a/programs/fileio.c b/programs/fileio.c index c137c78b..d1c4a7c7 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -354,19 +354,21 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha size_t readSize, decodedSize; /* Fill input buffer */ + if (toRead > inBuffSize) + EXM_THROW(34, "too large block"); readSize = fread(inBuff, 1, toRead, finput); if (readSize != toRead) - EXM_THROW(34, "Read error"); + EXM_THROW(35, "Read error"); /* Decode block */ decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); - if (ZSTD_isError(decodedSize)) EXM_THROW(35, "Decoding error : input corrupted"); + if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted"); if (decodedSize) /* not a header */ { /* Write block */ sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(36, "Write error : unable to write data block to destination file"); + if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); filesize += decodedSize; op += decodedSize; if (op==oend) op = outBuff; From 86ce906799972fa3ddaccfe111311b2cae299759 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 24 Aug 2015 20:29:17 +0100 Subject: [PATCH 3/3] Updated readme of fuzzer tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c3db8e2..23f822ee 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Another property zstd is developed for is configurable memory requirement, with Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entrop library](https://github.com/Cyan4973/FiniteStateEntropy). -Zstd has not yet reached "stable" status. Specifically, it doesn't guarantee yet that its current compressed format will remain stable and supported in future versions of the library. However, its current behavior is pretty solid, and able to withstand many hazards situations, including invalid input. Therefore, you can safely test zstd into controlled environments. +Zstd has not yet reached "stable" status. Specifically, it doesn't guarantee yet that its current compressed format will remain stable and supported in future versions. It may still change to adapt further optimizations still being investigated. However, the library starts to be pretty robust, able to withstand hazards situations, including invalid input. The library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), using both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, you can now safely test zstd, even within production environments. "Stable Format" is projected sometimes early 2016.