facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3040 from facebook/fix44239

Browse Source 
fix 44239
This commit is contained in:
Yann Collet 2022-02-01 13:33:01 -08:00 committed by GitHub
commit 529a5879bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/compress/zstd_compress.c
View File

@ -2595,7 +2595,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
entropyWorkspace = count + (MaxSeq + 1); entropyWorkspace = count + (MaxSeq + 1);
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq); DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE); assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
@ -2639,7 +2639,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
return (size_t)(op - ostart); return (size_t)(op - ostart);
} }
{ BYTE* seqHead = op++; { BYTE* const seqHead = op++;
/* build stats for sequences */ /* build stats for sequences */
const ZSTD_symbolEncodingTypeStats_t stats = const ZSTD_symbolEncodingTypeStats_t stats =
ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
@ -2702,15 +2702,17 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
*/ */
if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) {
DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
return 0; /* block not compressed */ return 0; /* block not compressed */
}
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
/* Check compressibility */ /* Check compressibility */
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */ if (cSize >= maxCSize) return 0; /* block not compressed */
} }
DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
return cSize; return cSize;
} }
@ -6159,6 +6161,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
continue; continue;
} }
RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
&cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
&cctx->appliedParams, &cctx->appliedParams,
@ -6182,11 +6185,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
if (compressedSeqsSize == 0) { if (compressedSeqsSize == 0) {
/* ZSTD_noCompressBlock writes the block header as well */ /* ZSTD_noCompressBlock writes the block header as well */
cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed");
DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
} else if (compressedSeqsSize == 1) { } else if (compressedSeqsSize == 1) {
cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed");
DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
} else { } else {
U32 cBlockHeader; U32 cBlockHeader;
@ -6203,7 +6206,6 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
} }
cSize += cBlockSize; cSize += cBlockSize;
DEBUGLOG(5, "cSize running total: %zu", cSize);
if (lastBlock) { if (lastBlock) {
break; break;
@ -6214,6 +6216,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
dstCapacity -= cBlockSize; dstCapacity -= cBlockSize;
cctx->isFirstBlock = 0; cctx->isFirstBlock = 0;
} }
DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
} }
DEBUGLOG(4, "cSize final total: %zu", cSize); DEBUGLOG(4, "cSize final total: %zu", cSize);
@ -6231,7 +6234,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
size_t frameHeaderSize = 0; size_t frameHeaderSize = 0;
/* Transparent initialization stage, same as compressStream2() */ /* Transparent initialization stage, same as compressStream2() */
DEBUGLOG(3, "ZSTD_compressSequences()"); DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
assert(cctx != NULL); assert(cctx != NULL);
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
/* Begin writing output, starting with frame header */ /* Begin writing output, starting with frame header */
@ -6259,7 +6262,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
cSize += 4; cSize += 4;
} }
DEBUGLOG(3, "Final compressed size: %zu", cSize); DEBUGLOG(4, "Final compressed size: %zu", cSize);
return cSize; return cSize;
} }

7
lib/compress/zstd_compress_internal.h
View File

@ -513,9 +513,11 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
/* ZSTD_noCompressBlock() : /* ZSTD_noCompressBlock() :
* Writes uncompressed block to dst buffer from given src. * Writes uncompressed block to dst buffer from given src.
* Returns the size of the block */ * Returns the size of the block */
MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) MEM_STATIC size_t
ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
{ {
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity);
RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
dstSize_tooSmall, "dst buf too small for uncompressed block"); dstSize_tooSmall, "dst buf too small for uncompressed block");
MEM_writeLE24(dst, cBlockHeader24); MEM_writeLE24(dst, cBlockHeader24);
@ -523,7 +525,8 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi
return ZSTD_blockHeaderSize + srcSize; return ZSTD_blockHeaderSize + srcSize;
} }
MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) MEM_STATIC size_t
ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
{ {
BYTE* const op = (BYTE*)dst; BYTE* const op = (BYTE*)dst;
U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);

6
lib/compress/zstd_compress_literals.c
View File

@ -41,6 +41,8 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
switch(flSize) switch(flSize)
@ -106,8 +108,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
symbolEncodingType_e hType = set_compressed; symbolEncodingType_e hType = set_compressed;
size_t cLitSize; size_t cLitSize;
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
disableLiteralCompression, (U32)srcSize); disableLiteralCompression, (U32)srcSize, dstCapacity);
DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));