facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Work around bug in zstd decoder (#1147)

Browse Source 
Work around bug in zstd decoder

Pull request #1144 exercised a new path in the zstd decoder that proved to
be buggy. Avoid the extremely rare bug by emitting an uncompressed block.
This commit is contained in:
Nick Terrell 2018-05-23 18:02:30 -07:00 committed by GitHub
parent 712318a244
commit 06b70179da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/compress/zstd_compress.c
View File

@ -1983,6 +1983,7 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
BYTE* op = ostart; BYTE* op = ostart;
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* seqHead; BYTE* seqHead;
BYTE* lastNCount = NULL;
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
@ -2033,6 +2034,8 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable),
workspace, HUF_WORKSPACE_SIZE); workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize; if (ZSTD_isError(countSize)) return countSize;
if (LLtype == set_compressed)
lastNCount = op;
op += countSize; op += countSize;
} } } }
/* build CTable for Offsets */ /* build CTable for Offsets */
@ -2049,6 +2052,8 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable),
workspace, HUF_WORKSPACE_SIZE); workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize; if (ZSTD_isError(countSize)) return countSize;
if (Offtype == set_compressed)
lastNCount = op;
op += countSize; op += countSize;
} } } }
/* build CTable for MatchLengths */ /* build CTable for MatchLengths */
@ -2063,6 +2068,8 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable),
workspace, HUF_WORKSPACE_SIZE); workspace, HUF_WORKSPACE_SIZE);
if (ZSTD_isError(countSize)) return countSize; if (ZSTD_isError(countSize)) return countSize;
if (MLtype == set_compressed)
lastNCount = op;
op += countSize; op += countSize;
} } } }
@ -2077,6 +2084,21 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
longOffsets, bmi2); longOffsets, bmi2);
if (ZSTD_isError(bitstreamSize)) return bitstreamSize; if (ZSTD_isError(bitstreamSize)) return bitstreamSize;
op += bitstreamSize; op += bitstreamSize;
/* zstd versions <= 1.3.4 mistakenly report corruption when
* FSE_readNCount() recieves a buffer < 4 bytes.
* Fixed by https://github.com/facebook/zstd/pull/1146.
* This can happen when the last set_compressed table present is 2
* bytes and the bitstream is only one byte.
* In this exceedingly rare case, we will simply emit an uncompressed
* block, since it isn't worth optimizing.
*/
if (lastNCount && (op - lastNCount) < 4) {
/* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
assert(op - lastNCount == 3);
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
"emitting an uncompressed block.");
return 0;
}
} }
return op - ostart; return op - ostart;
@ -2092,6 +2114,7 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
size_t const cSize = ZSTD_compressSequences_internal( size_t const cSize = ZSTD_compressSequences_internal(
seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity,
workspace, bmi2); workspace, bmi2);
if (cSize == 0) return 0;
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
*/ */