facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Various minor logical refactors to improve clarity

Browse Source
This commit is contained in:
senhuang42 2020-11-02 09:40:26 -05:00
parent e5fe485dcc
commit 1a449688fd
3 changed files with 35 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
lib/compress/zstd_compress.c
View File

@ -2342,7 +2342,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */ if (cSize >= maxCSize) return 0; /* block not compressed */
} }
DEBUGLOG(4, "compressSequences: %u\n", cSize); DEBUGLOG(4, "compressSequences cSize: %u\n", cSize);
return cSize; return cSize;
} }
@ -2660,10 +2660,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
zc->bmi2); zc->bmi2);
DEBUGLOG(4, "cSize compressed seqs: %u\n", cSize); DEBUGLOG(4, "cSize compressed seqs: %u\n", cSize);
if (cSize == 1) {
printf("Csize was 1 for some reason\n");
exit(1);
}
if (zc->seqCollector.collectSequences) { if (zc->seqCollector.collectSequences) {
ZSTD_copyBlockSequences(zc); ZSTD_copyBlockSequences(zc);
@ -2798,7 +2794,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
} }
void printBits(size_t const size, void const * const ptr) /*void printBits(size_t const size, void const * const ptr)
{ {
unsigned char *b = (unsigned char*) ptr; unsigned char *b = (unsigned char*) ptr;
unsigned char byte; unsigned char byte;
@ -2812,7 +2808,7 @@ void printBits(size_t const size, void const * const ptr)
printf("\n"); printf("\n");
} }
puts(""); puts("");
} }*/
/*! ZSTD_compress_frameChunk() : /*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks. * Compress a chunk of data into one or multiple blocks.
@ -2942,7 +2938,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
} }
printBits(pos, op); //printBits(pos, op);
DEBUGLOG(4, "frame header size: %u", pos); DEBUGLOG(4, "frame header size: %u", pos);
return pos; return pos;
} }
@ -4519,7 +4515,6 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n
U32 endPosInSequence = sequenceRange->endPosInSequence + nbBytes; U32 endPosInSequence = sequenceRange->endPosInSequence + nbBytes;
DEBUGLOG(4, "ZSTD_updateSequenceRange: startidx %u startpos: %u endidx: %u endpos: %u", DEBUGLOG(4, "ZSTD_updateSequenceRange: startidx %u startpos: %u endidx: %u endpos: %u",
sequenceRange->startIdx, sequenceRange->startPosInSequence, sequenceRange->endIdx, sequenceRange->endPosInSequence); sequenceRange->startIdx, sequenceRange->startPosInSequence, sequenceRange->endIdx, sequenceRange->endPosInSequence);
DEBUGLOG(4, "startidx: (of: %u ml: %u ll: %u", inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
while (endPosInSequence && idx < inSeqsSize) { while (endPosInSequence && idx < inSeqsSize) {
ZSTD_Sequence currSeq = inSeqs[idx]; ZSTD_Sequence currSeq = inSeqs[idx];
if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
@ -4552,7 +4547,7 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n
static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRange* seqRange, static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRange* seqRange,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize) { const void* src, size_t srcSize) {
DEBUGLOG(4, "ZSTD_copySequencesToSeqStore: numSeqs: %zu", inSeqsSize); DEBUGLOG(4, "ZSTD_copySequencesToSeqStore: numSeqs: %zu srcSize: %zu", inSeqsSize, srcSize);
size_t idx = seqRange->startIdx; size_t idx = seqRange->startIdx;
BYTE const* istart = (BYTE const*)src; BYTE const* istart = (BYTE const*)src;
BYTE const* ip = (BYTE const*)src; BYTE const* ip = (BYTE const*)src;
@ -4563,16 +4558,11 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
U32 litLength = inSeqs[idx].litLength; U32 litLength = inSeqs[idx].litLength;
U32 matchLength = inSeqs[idx].matchLength; U32 matchLength = inSeqs[idx].matchLength;
U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE; U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE;
//DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength, offCode, inSeqs[idx].rep);
/* Adjust litLength and matchLength for the sequence at startIdx */ /* Adjust litLength and matchLength for the sequence at startIdx */
if (idx == seqRange->startIdx) { if (idx == seqRange->startIdx) {
U32 posInSequence = seqRange->startPosInSequence; U32 posInSequence = seqRange->startPosInSequence;
if (posInSequence != 0) { DEBUGLOG(4, "At startIdx: idx: %u PIS: %u", idx, posInSequence);
DEBUGLOG(4, "nope\n");
exit(1);
}
DEBUGLOG(4, "Reached startIdx. idx: %u PIS: %u", idx, posInSequence);
assert(posInSequence <= litLength + matchLength); assert(posInSequence <= litLength + matchLength);
if (posInSequence >= litLength) { if (posInSequence >= litLength) {
litLength = 0; litLength = 0;
@ -4585,23 +4575,18 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
DEBUGLOG(4, "start idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode); DEBUGLOG(4, "start idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode);
RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small! Start Idx"); RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small! Start Idx");
} }
DEBUGLOG(4, "final ll:%u ml: %u", litLength, matchLength); DEBUGLOG(4, "startIdx seq finalized: ll: %u ml: %u", litLength, matchLength);
} }
/* Adjust litLength and matchLength for the sequence at endIdx */ /* Adjust litLength and matchLength for the sequence at endIdx */
if (idx == seqRange->endIdx) { if (idx == seqRange->endIdx) {
U32 posInSequence = seqRange->endPosInSequence; U32 posInSequence = seqRange->endPosInSequence;
if (posInSequence != 0) {
DEBUGLOG(4, "nope\n");
exit(1);
}
DEBUGLOG(4, "Reached endIdx. idx: %u PIS: %u", idx, posInSequence); DEBUGLOG(4, "Reached endIdx. idx: %u PIS: %u", idx, posInSequence);
if (posInSequence == 0) { if (posInSequence == 0) {
if (inSeqs[seqRange->endIdx - 1].matchLength != 0 || inSeqs[seqRange->endIdx - 1].matchLength != 0) { if (inSeqs[seqRange->endIdx - 1].matchLength != 0 || inSeqs[seqRange->endIdx - 1].matchLength != 0) {
printf("Contract violated\n"); printf("Contract violated\n");
exit(1); exit(1);
} }
// this will be our new start idx
return 0; return 0;
} }
assert(posInSequence <= litLength + matchLength); assert(posInSequence <= litLength + matchLength);
@ -4659,20 +4644,17 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
size_t origDstCapacity = dstCapacity; size_t origDstCapacity = dstCapacity;
DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %u, inSeqsSize: %u", srcSize, inSeqsSize); DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
/* Derive the appropriate block size */
BYTE const* ip = (BYTE const*)src; BYTE const* ip = (BYTE const*)src;
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
while (remaining) { while (remaining) {
DEBUGLOG(4, "--NEW BLOCK--");
U32 cBlockSize; U32 cBlockSize;
lastBlock = remaining <= cctx->blockSize; lastBlock = remaining <= cctx->blockSize;
blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize;
cctx->seqStore = baseSeqStore; cctx->seqStore = baseSeqStore;
DEBUGLOG(4, "Working on new block. Blocksize: %u", blockSize);
DEBUGLOG(4, "blockSize: %u", blockSize);
/* Skip over uncompressible blocks */ /* Skip over uncompressible blocks */
if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
DEBUGLOG(4, "block too small: %u", blockSize); DEBUGLOG(4, "block too small: %u", blockSize);
@ -4694,20 +4676,16 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
blockSize, blockSize,
cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
cctx->bmi2); cctx->bmi2);
FORWARD_IF_ERROR(compressedSeqsSize, "Compressing block failed");
DEBUGLOG(4, "Compressed sequences size : %u", compressedSeqsSize); DEBUGLOG(4, "Compressed sequences size : %u", compressedSeqsSize);
if (ZSTD_isError(compressedSeqsSize)) { if (compressedSeqsSize == 0) {
DEBUGLOG(4, "ERROR"); /* ZSTD_noCompressBlock writes the block header as well */
exit(1);
return compressedSeqsSize;
} else if (compressedSeqsSize == 0) {
DEBUGLOG(4, "NO compress BLOCK");
/* ZSTD_noCompressBlock writes block header as well */
cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
cSize += cBlockSize; FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
DEBUGLOG(4, "Block uncompressible, writing out nocompress block, size: %u", cBlockSize);
} else { } else {
cSize += compressedSeqsSize; /* Error checking and repcodes update */
/* Error checking */
if (!ZSTD_isError(compressedSeqsSize) && compressedSeqsSize > 1) { if (!ZSTD_isError(compressedSeqsSize) && compressedSeqsSize > 1) {
ZSTD_confirmRepcodesAndEntropyTables(cctx); ZSTD_confirmRepcodesAndEntropyTables(cctx);
} }
@ -4715,25 +4693,18 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
U32 const cBlockHeader = compressedSeqsSize == 1 ? U32 const cBlockHeader = compressedSeqsSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3): lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3):
lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
//printBits(ZSTD_blockHeaderSize, &cBlockHeader);
MEM_writeLE24(op, cBlockHeader); MEM_writeLE24(op, cBlockHeader);
cSize += ZSTD_blockHeaderSize;
DEBUGLOG(4, "Block header: %u", cBlockHeader);
/*DEBUGLOG(3, "block header type: %u", ((cBlockHeader >> 1) & 3));
if (((cBlockHeader >> 1) & 3) != 2) {
exit(1);
}*/
DEBUGLOG(4, "typical block, size: %u", compressedSeqsSize + ZSTD_blockHeaderSize);
cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
DEBUGLOG(4, "Writing out compressed block, size: %u", cBlockSize);
if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
//if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
//cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
} }
cSize += cBlockSize;
DEBUGLOG(4, "cumulative cSize: %u", cSize); DEBUGLOG(4, "cumulative cSize: %u", cSize);
if (lastBlock) { if (lastBlock) {
DEBUGLOG(4, "reached last block, breaking"); DEBUGLOG(4, "reached last block, stopping");
break; break;
} else { } else {
ip += blockSize; ip += blockSize;
@ -4751,12 +4722,14 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
const void* src, size_t srcSize, int compressionLevel) { const void* src, size_t srcSize, int compressionLevel) {
DEBUGLOG(4, "ZSTD_compressSequences_ext()"); DEBUGLOG(4, "ZSTD_compressSequences_ext()");
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t cSize = 0; size_t cSize = 0;
size_t compressedBlocksSize = 0;
size_t frameHeaderSize = 0; size_t frameHeaderSize = 0;
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
/* Initialization stage */
{ {
ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_CCtx_params params = cctx->requestedParams;
ZSTD_prefixDict const prefixDict = cctx->prefixDict; ZSTD_prefixDict const prefixDict = cctx->prefixDict;
@ -4785,10 +4758,9 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
ZSTDb_buffered) , ""); ZSTDb_buffered) , "");
assert(cctx->appliedParams.nbWorkers == 0); assert(cctx->appliedParams.nbWorkers == 0);
} }
DEBUGLOG(4, "blocksize: %u\n", cctx->blockSize); DEBUGLOG(4, "CCtx blockSize: %zu\n", cctx->blockSize);
if (dstCapacity < ZSTD_compressBound(srcSize)) if (dstCapacity < ZSTD_compressBound(srcSize))
RETURN_ERROR(dstSize_tooSmall, "Destination buffer too small!"); RETURN_ERROR(dstSize_tooSmall, "Destination buffer too small!");
DEBUGLOG(4, "SeqStore: maxNbSeq: %u, maxNbLits: %u", cctx->seqStore.maxNbSeq, cctx->seqStore.maxNbLit);
frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID);
op += frameHeaderSize; op += frameHeaderSize;
@ -4803,11 +4775,15 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
} }
/* cSize includes block header size and compressed sequences size */ /* cSize includes block header size and compressed sequences size */
cSize += ZSTD_compressSequences_ext_internal(op, dstCapacity, compressedBlocksSize = ZSTD_compressSequences_ext_internal(op, dstCapacity,
cctx, inSeqs, inSeqsSize, cctx, inSeqs, inSeqsSize,
src, srcSize); src, srcSize);
DEBUGLOG(4, "cSize after compressSequences_internal: %u\n", cSize); if (ZSTD_isError(compressedBlocksSize)) {
dstCapacity -= cSize; return compressedBlocksSize;
}
cSize += compressedBlocksSize;
dstCapacity -= compressedBlocksSize;
DEBUGLOG(4, "cSize after compressSequences_internal: %zu\n", cSize);
if (cctx->appliedParams.fParams.checksumFlag) { if (cctx->appliedParams.fParams.checksumFlag) {
U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
@ -4817,7 +4793,7 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
cSize += 4; cSize += 4;
} }
DEBUGLOG(4, "Final compressed size: %u\n", cSize); DEBUGLOG(4, "Final compressed size: %zu\n", cSize);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
return cSize; return cSize;
} }

9
lib/decompress/zstd_decompress.c
View File

@ -497,6 +497,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
if (ret > 0) if (ret > 0)
return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
} }
ip += zfh.headerSize; ip += zfh.headerSize;
remainingSize -= zfh.headerSize; remainingSize -= zfh.headerSize;
@ -645,13 +646,8 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
while (1) { while (1) {
size_t decodedSize; size_t decodedSize;
blockProperties_t blockProperties; blockProperties_t blockProperties;
printf("Getting blocksize\n");
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
printf("BlockSize: %u\n", cBlockSize); if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (ZSTD_isError(cBlockSize)) {
printf("Errored\n");
return cBlockSize;
}
ip += ZSTD_blockHeaderSize; ip += ZSTD_blockHeaderSize;
remainingSrcSize -= ZSTD_blockHeaderSize; remainingSrcSize -= ZSTD_blockHeaderSize;
@ -1621,7 +1617,6 @@ static size_t ZSTD_decompressContinueStream(
zds->streamStage = zdss_flush; zds->streamStage = zdss_flush;
} }
} else { } else {
/* Write directly into the output buffer */ /* Write directly into the output buffer */
size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);

23
lib/decompress/zstd_decompress_block.c
View File

@ -46,20 +46,6 @@
**********************************************************/ **********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
void printBits1(size_t const size, void const * const ptr)
{
unsigned char *b = (unsigned char*) ptr;
unsigned char byte;
int i, j;
for (i = size-1; i >= 0; i--) {
for (j = 7; j >= 0; j--) {
byte = (b[i] >> j) & 1;
printf("%u", byte);
}
printf("\n");
}
}
/*-************************************************************* /*-*************************************************************
* Block decoding * Block decoding
@ -70,7 +56,6 @@ void printBits1(size_t const size, void const * const ptr)
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
blockProperties_t* bpPtr) blockProperties_t* bpPtr)
{ {
printf("getcblockSize: srcSize: %u\n", srcSize);
RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
{ U32 const cBlockHeader = MEM_readLE24(src); { U32 const cBlockHeader = MEM_readLE24(src);
@ -78,12 +63,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
bpPtr->lastBlock = cBlockHeader & 1; bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */ bpPtr->origSize = cSize; /* only useful for RLE */
printBits1(3, &cBlockHeader); if (bpPtr->blockType == bt_rle) return 1;
if (bpPtr->blockType == bt_rle) {
printf("RLE BLOCK FOUND\n");
exit(1);
return 1;
}
RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
return cSize; return cSize;
} }
@ -439,7 +419,6 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
* our buffer to handle the over-write. * our buffer to handle the over-write.
*/ */
{ {
U64 const add = 0x0101010101010101ull; U64 const add = 0x0101010101010101ull;
size_t pos = 0; size_t pos = 0;
U64 sv = 0; U64 sv = 0;