facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #782 from terrelln/dstSizeTooSmall 

Fix compression failure on incompressible data
dev
Yann Collet 2017-08-07 14:52:02 -07:00 committed by GitHub
parent a1fe656165 308047eb5d
commit d9f2893eb9
1 changed files with 34 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
lib/compress/zstd_compress.c
View File

@ -942,7 +942,7 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */ else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */
} }
if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) { if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
entropy->hufCTable_repeatMode = HUF_repeat_none; entropy->hufCTable_repeatMode = HUF_repeat_none;
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
} }
@ -1156,11 +1156,10 @@ MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
} }
} }
MEM_STATIC size_t ZSTD_compressSequences (seqStore_t* seqStorePtr, MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_entropyCTables_t* entropy, ZSTD_entropyCTables_t* entropy,
ZSTD_compressionParameters const* cParams, ZSTD_compressionParameters const* cParams,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity)
size_t srcSize)
{ {
const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN; const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN;
U32 count[MaxSeq+1]; U32 count[MaxSeq+1];
@ -1195,7 +1194,7 @@ MEM_STATIC size_t ZSTD_compressSequences (seqStore_t* seqStorePtr,
if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq;
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
if (nbSeq==0) goto _check_compressibility; if (nbSeq==0) return op - ostart;
/* seqHead : flags for FSE encoding type */ /* seqHead : flags for FSE encoding type */
seqHead = op++; seqHead = op++;
@ -1244,23 +1243,40 @@ MEM_STATIC size_t ZSTD_compressSequences (seqStore_t* seqStorePtr,
op += streamSize; op += streamSize;
} }
return op - ostart;
}
/* check compressibility */ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
_check_compressibility: ZSTD_entropyCTables_t* entropy,
{ size_t const minGain = ZSTD_minGain(srcSize); ZSTD_compressionParameters const* cParams,
void* dst, size_t dstCapacity,
size_t srcSize)
{
size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams,
dst, dstCapacity);
size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain; size_t const maxCSize = srcSize - minGain;
if ((size_t)(op-ostart) >= maxCSize) { /* If the srcSize <= dstCapacity, then there is enough space to write a
* raw uncompressed block. Since we ran out of space, the block must not
* be compressible, so fall back to a raw uncompressed block.
*/
int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
if (ZSTD_isError(cSize) && !uncompressibleError)
return cSize;
/* Check compressibility */
if (cSize >= maxCSize || uncompressibleError) {
entropy->hufCTable_repeatMode = HUF_repeat_none; entropy->hufCTable_repeatMode = HUF_repeat_none;
entropy->offcode_repeatMode = FSE_repeat_none; entropy->offcode_repeatMode = FSE_repeat_none;
entropy->matchlength_repeatMode = FSE_repeat_none; entropy->matchlength_repeatMode = FSE_repeat_none;
entropy->litlength_repeatMode = FSE_repeat_none; entropy->litlength_repeatMode = FSE_repeat_none;
return 0; return 0;
} } }
assert(!ZSTD_isError(cSize));
/* confirm repcodes */ /* confirm repcodes */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
return cSize;
return op - ostart;
} }