From 62c746dcf90c9f69f404df08533795d72083361d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 28 Nov 2017 12:06:48 -0500 Subject: [PATCH 1/3] Add Test on LZ4 Format Input Buffer Overrun --- tests/Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 11d55a75..f944fd5a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -374,9 +374,9 @@ test-pool: poolTests test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4 test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4 -test-lz4: zstd decodecorpus - ln -s $(PRGDIR)/zstd lz4 - ln -s $(PRGDIR)/zstd unlz4 +test-lz4: zstd decodecorpus datagen + [ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4 + [ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4 ./decodecorpus -ptmp # lz4 -> zstd @@ -402,6 +402,8 @@ test-lz4: zstd decodecorpus $(ZSTD) -d | \ cmp - tmp + ./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null + rm tmp lz4 unlz4 endif From baff9dd15eb65eaf54f0dee9ee5fadc9ce16d6ab Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 17 Oct 2017 01:19:29 -0400 Subject: [PATCH 2/3] Fix LZ4 Compression Buffer Overflow Fixes issue where, when `zstd --format=lz4` is fed an input larger than 128KB, the read overruns the input buffer. This changes Zstd to use LZ4 with chained 64KB blocks. This is technically a breaking change in that some third party LZ4 implementations may not support linked blocks. However, progress should not be allowed to be stopped by such petty concerns as backwards compatibility! --- programs/fileio.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8bee6086..eb004f91 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -616,11 +616,16 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, #endif #ifdef ZSTD_LZ4COMPRESS +#if LZ4_VERSION_NUMBER <= 10600 +#define LZ4F_blockLinked blockLinked +#define LZ4F_max64KB max64KB +#endif static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } static unsigned long long FIO_compressLz4Frame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) { + const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); unsigned long long inFileSize = 0, outFileSize = 0; LZ4F_preferences_t prefs; @@ -632,21 +637,19 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, memset(&prefs, 0, sizeof(prefs)); -#if LZ4_VERSION_NUMBER <= 10600 -#define LZ4F_blockIndependent blockIndependent -#define LZ4F_max4MB max4MB -#endif + assert(blockSize <= ress->srcBufferSize); + assert(LZ4F_compressBound(blockSize) <= ress->dstBufferSize); prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */ - prefs.frameInfo.blockSizeID = LZ4F_max4MB; + prefs.frameInfo.blockMode = LZ4F_blockLinked; + prefs.frameInfo.blockSizeID = LZ4F_max64KB; prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; #if LZ4_VERSION_NUMBER >= 10600 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; #endif - { size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB); + { size_t readSize; size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); if (LZ4F_isError(headerSize)) From f5747afe0fffa5fb1c993c5366d6a6c7f73783a0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Nov 2017 15:02:48 -0800 Subject: [PATCH 3/3] add integer overflow trapping to debug builds doc: https://gcc.gnu.org/onlinedocs/gcc-4.0.2/gcc/Code-Gen-Options.html --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 11d55a75..89a109c7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -32,7 +32,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls + -Wredundant-decls -ftrapv CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)