From 1a449688fdf44a3c88c20eb69a848ee2241485d9 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 2 Nov 2020 09:40:26 -0500 Subject: [PATCH] Various minor logical refactors to improve clarity --- lib/compress/zstd_compress.c | 88 ++++++++++---------------- lib/decompress/zstd_decompress.c | 9 +-- lib/decompress/zstd_decompress_block.c | 23 +------ 3 files changed, 35 insertions(+), 85 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 31a5f85a..6922e954 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2342,7 +2342,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); if (cSize >= maxCSize) return 0; /* block not compressed */ } - DEBUGLOG(4, "compressSequences: %u\n", cSize); + DEBUGLOG(4, "compressSequences cSize: %u\n", cSize); return cSize; } @@ -2660,10 +2660,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); DEBUGLOG(4, "cSize compressed seqs: %u\n", cSize); - if (cSize == 1) { - printf("Csize was 1 for some reason\n"); - exit(1); - } if (zc->seqCollector.collectSequences) { ZSTD_copyBlockSequences(zc); @@ -2798,7 +2794,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, } -void printBits(size_t const size, void const * const ptr) +/*void printBits(size_t const size, void const * const ptr) { unsigned char *b = (unsigned char*) ptr; unsigned char byte; @@ -2812,7 +2808,7 @@ void printBits(size_t const size, void const * const ptr) printf("\n"); } puts(""); -} +}*/ /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. @@ -2942,7 +2938,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; } - printBits(pos, op); + //printBits(pos, op); DEBUGLOG(4, "frame header size: %u", pos); return pos; } @@ -4519,7 +4515,6 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n U32 endPosInSequence = sequenceRange->endPosInSequence + nbBytes; DEBUGLOG(4, "ZSTD_updateSequenceRange: startidx %u startpos: %u endidx: %u endpos: %u", sequenceRange->startIdx, sequenceRange->startPosInSequence, sequenceRange->endIdx, sequenceRange->endPosInSequence); - DEBUGLOG(4, "startidx: (of: %u ml: %u ll: %u", inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); while (endPosInSequence && idx < inSeqsSize) { ZSTD_Sequence currSeq = inSeqs[idx]; if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { @@ -4552,7 +4547,7 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRange* seqRange, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const void* src, size_t srcSize) { - DEBUGLOG(4, "ZSTD_copySequencesToSeqStore: numSeqs: %zu", inSeqsSize); + DEBUGLOG(4, "ZSTD_copySequencesToSeqStore: numSeqs: %zu srcSize: %zu", inSeqsSize, srcSize); size_t idx = seqRange->startIdx; BYTE const* istart = (BYTE const*)src; BYTE const* ip = (BYTE const*)src; @@ -4563,16 +4558,11 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan U32 litLength = inSeqs[idx].litLength; U32 matchLength = inSeqs[idx].matchLength; U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE; - //DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength, offCode, inSeqs[idx].rep); /* Adjust litLength and matchLength for the sequence at startIdx */ if (idx == seqRange->startIdx) { U32 posInSequence = seqRange->startPosInSequence; - if (posInSequence != 0) { - DEBUGLOG(4, "nope\n"); - exit(1); - } - DEBUGLOG(4, "Reached startIdx. idx: %u PIS: %u", idx, posInSequence); + DEBUGLOG(4, "At startIdx: idx: %u PIS: %u", idx, posInSequence); assert(posInSequence <= litLength + matchLength); if (posInSequence >= litLength) { litLength = 0; @@ -4585,23 +4575,18 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan DEBUGLOG(4, "start idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode); RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small! Start Idx"); } - DEBUGLOG(4, "final ll:%u ml: %u", litLength, matchLength); + DEBUGLOG(4, "startIdx seq finalized: ll: %u ml: %u", litLength, matchLength); } /* Adjust litLength and matchLength for the sequence at endIdx */ if (idx == seqRange->endIdx) { U32 posInSequence = seqRange->endPosInSequence; - if (posInSequence != 0) { - DEBUGLOG(4, "nope\n"); - exit(1); - } DEBUGLOG(4, "Reached endIdx. idx: %u PIS: %u", idx, posInSequence); if (posInSequence == 0) { if (inSeqs[seqRange->endIdx - 1].matchLength != 0 || inSeqs[seqRange->endIdx - 1].matchLength != 0) { printf("Contract violated\n"); exit(1); } - // this will be our new start idx return 0; } assert(posInSequence <= litLength + matchLength); @@ -4659,20 +4644,17 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, size_t origDstCapacity = dstCapacity; - DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %u, inSeqsSize: %u", srcSize, inSeqsSize); - /* Derive the appropriate block size */ - + DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); BYTE const* ip = (BYTE const*)src; BYTE* op = (BYTE*)dst; while (remaining) { - DEBUGLOG(4, "--NEW BLOCK--"); U32 cBlockSize; lastBlock = remaining <= cctx->blockSize; blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; cctx->seqStore = baseSeqStore; + DEBUGLOG(4, "Working on new block. Blocksize: %u", blockSize); - DEBUGLOG(4, "blockSize: %u", blockSize); /* Skip over uncompressible blocks */ if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { DEBUGLOG(4, "block too small: %u", blockSize); @@ -4694,20 +4676,16 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, blockSize, cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing block failed"); DEBUGLOG(4, "Compressed sequences size : %u", compressedSeqsSize); - if (ZSTD_isError(compressedSeqsSize)) { - DEBUGLOG(4, "ERROR"); - exit(1); - return compressedSeqsSize; - } else if (compressedSeqsSize == 0) { - DEBUGLOG(4, "NO compress BLOCK"); - /* ZSTD_noCompressBlock writes block header as well */ + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - cSize += cBlockSize; + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(4, "Block uncompressible, writing out nocompress block, size: %u", cBlockSize); } else { - cSize += compressedSeqsSize; - /* Error checking */ + /* Error checking and repcodes update */ if (!ZSTD_isError(compressedSeqsSize) && compressedSeqsSize > 1) { ZSTD_confirmRepcodesAndEntropyTables(cctx); } @@ -4715,25 +4693,18 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, U32 const cBlockHeader = compressedSeqsSize == 1 ? lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3): lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); - //printBits(ZSTD_blockHeaderSize, &cBlockHeader); MEM_writeLE24(op, cBlockHeader); - cSize += ZSTD_blockHeaderSize; - DEBUGLOG(4, "Block header: %u", cBlockHeader); - /*DEBUGLOG(3, "block header type: %u", ((cBlockHeader >> 1) & 3)); - if (((cBlockHeader >> 1) & 3) != 2) { - exit(1); - }*/ - DEBUGLOG(4, "typical block, size: %u", compressedSeqsSize + ZSTD_blockHeaderSize); cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %u", cBlockSize); - - //if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - //cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; } + cSize += cBlockSize; DEBUGLOG(4, "cumulative cSize: %u", cSize); if (lastBlock) { - DEBUGLOG(4, "reached last block, breaking"); + DEBUGLOG(4, "reached last block, stopping"); break; } else { ip += blockSize; @@ -4751,12 +4722,14 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { DEBUGLOG(4, "ZSTD_compressSequences_ext()"); BYTE* op = (BYTE*)dst; - ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t cSize = 0; + size_t compressedBlocksSize = 0; size_t frameHeaderSize = 0; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + /* Initialization stage */ { ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; @@ -4785,10 +4758,9 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity, ZSTDb_buffered) , ""); assert(cctx->appliedParams.nbWorkers == 0); } - DEBUGLOG(4, "blocksize: %u\n", cctx->blockSize); + DEBUGLOG(4, "CCtx blockSize: %zu\n", cctx->blockSize); if (dstCapacity < ZSTD_compressBound(srcSize)) RETURN_ERROR(dstSize_tooSmall, "Destination buffer too small!"); - DEBUGLOG(4, "SeqStore: maxNbSeq: %u, maxNbLits: %u", cctx->seqStore.maxNbSeq, cctx->seqStore.maxNbLit); frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); op += frameHeaderSize; @@ -4803,11 +4775,15 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity, } /* cSize includes block header size and compressed sequences size */ - cSize += ZSTD_compressSequences_ext_internal(op, dstCapacity, + compressedBlocksSize = ZSTD_compressSequences_ext_internal(op, dstCapacity, cctx, inSeqs, inSeqsSize, src, srcSize); - DEBUGLOG(4, "cSize after compressSequences_internal: %u\n", cSize); - dstCapacity -= cSize; + if (ZSTD_isError(compressedBlocksSize)) { + return compressedBlocksSize; + } + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + DEBUGLOG(4, "cSize after compressSequences_internal: %zu\n", cSize); if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); @@ -4817,7 +4793,7 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity, cSize += 4; } - DEBUGLOG(4, "Final compressed size: %u\n", cSize); + DEBUGLOG(4, "Final compressed size: %zu\n", cSize); ZSTD_freeCCtx(cctx); return cSize; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index fc8772a5..21f846bc 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -497,6 +497,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize if (ret > 0) return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); } + ip += zfh.headerSize; remainingSize -= zfh.headerSize; @@ -645,13 +646,8 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, while (1) { size_t decodedSize; blockProperties_t blockProperties; - printf("Getting blocksize\n"); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); - printf("BlockSize: %u\n", cBlockSize); - if (ZSTD_isError(cBlockSize)) { - printf("Errored\n"); - return cBlockSize; - } + if (ZSTD_isError(cBlockSize)) return cBlockSize; ip += ZSTD_blockHeaderSize; remainingSrcSize -= ZSTD_blockHeaderSize; @@ -1621,7 +1617,6 @@ static size_t ZSTD_decompressContinueStream( zds->streamStage = zdss_flush; } } else { - /* Write directly into the output buffer */ size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index d4c7d383..bec82e85 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -46,20 +46,6 @@ **********************************************************/ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } -void printBits1(size_t const size, void const * const ptr) -{ - unsigned char *b = (unsigned char*) ptr; - unsigned char byte; - int i, j; - - for (i = size-1; i >= 0; i--) { - for (j = 7; j >= 0; j--) { - byte = (b[i] >> j) & 1; - printf("%u", byte); - } - printf("\n"); - } -} /*-************************************************************* * Block decoding @@ -70,7 +56,6 @@ void printBits1(size_t const size, void const * const ptr) size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) { - printf("getcblockSize: srcSize: %u\n", srcSize); RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); { U32 const cBlockHeader = MEM_readLE24(src); @@ -78,12 +63,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, bpPtr->lastBlock = cBlockHeader & 1; bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->origSize = cSize; /* only useful for RLE */ - printBits1(3, &cBlockHeader); - if (bpPtr->blockType == bt_rle) { - printf("RLE BLOCK FOUND\n"); - exit(1); - return 1; - } + if (bpPtr->blockType == bt_rle) return 1; RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); return cSize; } @@ -439,7 +419,6 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, * our buffer to handle the over-write. */ { - U64 const add = 0x0101010101010101ull; size_t pos = 0; U64 sv = 0;