facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for uncompressible blocks

dev
senhuang42 2020-10-29 10:24:45 -04:00
parent e04da68157
commit 75b01f34b9
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/compress/zstd_compress.c
View File

@ -2927,6 +2927,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
} }
printBits(pos, op); printBits(pos, op);
DEBUGLOG(4, "frame header size: %u", pos);
return pos; return pos;
} }
@ -4540,7 +4541,7 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
U32 litLength = inSeqs[idx].litLength; U32 litLength = inSeqs[idx].litLength;
U32 matchLength = inSeqs[idx].matchLength; U32 matchLength = inSeqs[idx].matchLength;
U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE; U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE;
//DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode); DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode);
/* Adjust litLength and matchLength for the sequence at startIdx */ /* Adjust litLength and matchLength for the sequence at startIdx */
if (idx == seqRange->startIdx) { if (idx == seqRange->startIdx) {
@ -4609,8 +4610,15 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize) { const void* src, size_t srcSize) {
U32 cSize; U32 cSize;
U32 lastBlock = 1; U32 lastBlock;
ZSTD_sequenceRange seqRange = {0, 0, 0, 0}; ZSTD_sequenceRange seqRange = {0, 0, 0, 0};
/* Derive the appropriate block size */
size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << cctx->appliedParams.cParams.windowLog), srcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
BYTE const* ip = (BYTE const*)src;
BYTE* op = (BYTE*)dst;
ZSTD_updateSequenceRange(&seqRange, srcSize, inSeqs, inSeqsSize); ZSTD_updateSequenceRange(&seqRange, srcSize, inSeqs, inSeqsSize);
ZSTD_copySequencesToSeqStore(cctx, &seqRange, inSeqs, inSeqsSize, src, srcSize); ZSTD_copySequencesToSeqStore(cctx, &seqRange, inSeqs, inSeqsSize, src, srcSize);
@ -4622,7 +4630,13 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
srcSize, srcSize,
cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
cctx->bmi2); cctx->bmi2);
if (ZSTD_isError(cSize)) {
printf("ERROR\n");
} else if (cSize == 0) {
DEBUGLOG(4, "NO compress BLOCK");
cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
return cSize;
}
DEBUGLOG(4, "Compressed sequences size : %u", cSize); DEBUGLOG(4, "Compressed sequences size : %u", cSize);
/* Error checking */ /* Error checking */
if (!ZSTD_isError(cSize) && cSize > 1) { if (!ZSTD_isError(cSize) && cSize > 1) {
@ -4698,6 +4712,7 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
cSize += ZSTD_compressSequences_ext_internal(op, dstCapacity, cSize += ZSTD_compressSequences_ext_internal(op, dstCapacity,
cctx, inSeqs, inSeqsSize, cctx, inSeqs, inSeqsSize,
src, srcSize); src, srcSize);
DEBUGLOG(4, "Final compressed size: %u\n", cSize);
dstCapacity -= cSize; dstCapacity -= cSize;
if (cctx->appliedParams.fParams.checksumFlag) { if (cctx->appliedParams.fParams.checksumFlag) {

1
lib/compress/zstd_compress_internal.h
View File

@ -432,6 +432,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi
dstSize_tooSmall, "dst buf too small for uncompressed block"); dstSize_tooSmall, "dst buf too small for uncompressed block");
MEM_writeLE24(dst, cBlockHeader24); MEM_writeLE24(dst, cBlockHeader24);
ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
DEBUGLOG(4, "nocompress: %u", ZSTD_blockHeaderSize + srcSize);
return ZSTD_blockHeaderSize + srcSize; return ZSTD_blockHeaderSize + srcSize;
} }