From 0633bf17c3dcbad8abb895dcb5bc508f23b98bf8 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 11 Jan 2021 08:30:16 -0500 Subject: [PATCH] Change 1.3.4 bugfix to be cross-compatible with superblocks and normal compression --- lib/compress/zstd_compress.c | 47 ++++++++++++------------- lib/compress/zstd_compress_internal.h | 2 +- lib/compress/zstd_compress_superblock.c | 4 +-- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 854a8e39..3be9e1df 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2208,7 +2208,7 @@ static int ZSTD_useBlockSplitting(const ZSTD_CCtx_params* cctxParams) */ MEM_STATIC size_t ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, - U32* LLtype, U32* Offtype, U32* MLtype, BYTE** lastNCount, + U32* LLtype, U32* Offtype, U32* MLtype, size_t* lastCountSize, const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, BYTE* dst, const BYTE* const dstEnd, ZSTD_strategy strategy, @@ -2252,7 +2252,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); if (*LLtype == set_compressed) - *lastNCount = op; + *lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2280,7 +2280,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); if (*Offtype == set_compressed) - *lastNCount = op; + *lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2306,7 +2306,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); if (*MLtype == set_compressed) - *lastNCount = op; + *lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2339,8 +2339,7 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - BYTE* lastNCount = NULL; - BYTE* seqHead; + size_t lastCountSize = 0; entropyWorkspace = count + (MaxSeq + 1); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); @@ -2385,15 +2384,14 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); return (size_t)(op - ostart); } - { U32 LLtype; U32 Offtype; U32 MLtype; - seqHead = op++; + BYTE* seqHead = op++; /* build stats for sequences */ entropyStatisticsSize = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, - &LLtype, &Offtype, &MLtype, &lastNCount, + &LLtype, &Offtype, &MLtype, &lastCountSize, &prevEntropy->fse, &nextEntropy->fse, op, oend, strategy, @@ -2421,9 +2419,9 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr, * In this exceedingly rare case, we will simply emit an uncompressed * block, since it isn't worth optimizing. */ - if (lastNCount && (op - lastNCount) < 4) { - /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(op - lastNCount == 3); + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " "emitting an uncompressed block."); return 0; @@ -2886,20 +2884,20 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, BYTE* const ostart = fseMetadata->fseTablesBuffer; BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); BYTE* op = ostart; - BYTE* lastNCount = NULL; size_t hSize; - U32 LLtype; - U32 Offtype; - U32 MLtype; + + /* ZSTD_buildSequencesStatistics() is guaranteed to overwrite these values */ + U32 LLtype = set_basic; + U32 Offtype = set_basic; + U32 MLtype = set_basic; DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); ZSTD_memset(workspace, 0, wkspSize); hSize = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, - &LLtype, &Offtype, &MLtype, &lastNCount, + &LLtype, &Offtype, &MLtype, &fseMetadata->lastCountSize, prevEntropy, nextEntropy, op, oend, strategy, workspace, wkspSize); - fseMetadata->lastNCount = lastNCount; fseMetadata->llType = (symbolEncodingType_e) LLtype; fseMetadata->ofType = (symbolEncodingType_e) Offtype; fseMetadata->mlType = (symbolEncodingType_e) MLtype; @@ -3104,8 +3102,8 @@ static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* seqStore) { static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, const seqStore_t* originalSeqStore, size_t startIdx, size_t endIdx) { BYTE* const litEnd = originalSeqStore->lit; - U32 literalsBytes; - U32 literalsBytesPreceding = 0; + size_t literalsBytes; + size_t literalsBytesPreceding = 0; *resultSeqStore = *originalSeqStore; if (startIdx > 0) { @@ -3118,7 +3116,7 @@ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, const seqStore_ if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { resultSeqStore->longLengthID = 0; } else { - resultSeqStore->longLengthPos -= startIdx; + resultSeqStore->longLengthPos -= (U32)startIdx; } } resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; @@ -3240,7 +3238,7 @@ static void ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx } if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); - splits->splitLocations[splits->idx] = midIdx; + splits->splitLocations[splits->idx] = (U32)midIdx; splits->idx++; ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); } @@ -3322,21 +3320,20 @@ static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, const void* src, size_t srcSize, U32 lastBlock) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; - size_t nbSeq; + U32 nbSeq; size_t cSize; DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); if (bss == ZSTDbss_noCompress) { - cSize = 0; if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); return cSize; } - nbSeq = (size_t)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); } assert(zc->appliedParams.splitBlocks == 1); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 6cfeb628..43252aee 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -107,7 +107,7 @@ typedef struct { symbolEncodingType_e mlType; BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; size_t fseTablesSize; - BYTE* lastNCount; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSequences_internal() */ } ZSTD_fseCTablesMetadata_t; typedef struct { diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c index 1612c7bc..e9452e33 100644 --- a/lib/compress/zstd_compress_superblock.c +++ b/lib/compress/zstd_compress_superblock.c @@ -500,9 +500,9 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables * block, since it isn't worth optimizing. */ #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (writeEntropy && fseMetadata->lastNCount && (op - fseMetadata->lastNCount) < 4) { + if (writeEntropy && fseMetadata->lastCountSize && (bitstreamSize + fseMetadata->lastCountSize) < 4) { /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(op - fseMetadata->lastNCount == 3); + assert(bitstreamSize + fseMetadata->lastCountSize == 3); DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " "emitting an uncompressed block."); return 0;