facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix LZ4 Compression Buffer Overflow

Browse Source 
Fixes issue where, when `zstd --format=lz4` is fed an input larger than 128KB,
the read overruns the input buffer. This changes Zstd to use LZ4 with chained
64KB blocks. This is technically a breaking change in that some third party
LZ4 implementations may not support linked blocks. However, progress should not
be allowed to be stopped by such petty concerns as backwards compatibility!
This commit is contained in:
W. Felix Handte 2017-10-17 01:19:29 -04:00
parent 62c746dcf9
commit baff9dd15e
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
programs/fileio.c
View File

@ -616,11 +616,16 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
#endif
#ifdef ZSTD_LZ4COMPRESS
#if LZ4_VERSION_NUMBER <= 10600
#define LZ4F_blockLinked blockLinked
#define LZ4F_max64KB max64KB
#endif
static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize)
{
const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
unsigned long long inFileSize = 0, outFileSize = 0;
LZ4F_preferences_t prefs;
@ -632,21 +637,19 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
memset(&prefs, 0, sizeof(prefs));
#if LZ4_VERSION_NUMBER <= 10600
#define LZ4F_blockIndependent blockIndependent
#define LZ4F_max4MB max4MB
#endif
assert(blockSize <= ress->srcBufferSize);
assert(LZ4F_compressBound(blockSize) <= ress->dstBufferSize);
prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
prefs.frameInfo.blockMode = LZ4F_blockLinked;
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag;
#if LZ4_VERSION_NUMBER >= 10600
prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
#endif
{ size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB);
{
size_t readSize;
size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs);
if (LZ4F_isError(headerSize))