facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix warnings and make validation enabled by default

Browse Source
This commit is contained in:
senhuang42 2020-11-16 11:47:27 -05:00
parent 55b90ef010
commit f6baad87d6
1 changed files with 11 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lib/compress/zstd_compress.c
View File

@ -4492,9 +4492,8 @@ typedef struct {
U64 posInSrc; /* Number of bytes given by sequences provided so far */ U64 posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_sequencePosition; } ZSTD_sequencePosition;
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
/* Returns a ZSTD error code if sequence is not valid */ /* Returns a ZSTD error code if sequence is not valid */
static size_t ZSTD_validateSequence(U32 offCode, U32 repCode, U32 matchLength, static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength,
size_t posInSrc, U32 windowLog, U32 dictSize) { size_t posInSrc, U32 windowLog, U32 dictSize) {
U32 offsetBound; U32 offsetBound;
U32 windowSize = 1 << windowLog; U32 windowSize = 1 << windowLog;
@ -4504,11 +4503,10 @@ static size_t ZSTD_validateSequence(U32 offCode, U32 repCode, U32 matchLength,
* window size. After output surpasses windowSize, we're limited to windowSize offsets again. * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
*/ */
offsetBound = posInSrc > windowSize ? windowSize : posInSrc + dictSize; offsetBound = posInSrc > windowSize ? windowSize : posInSrc + dictSize;
RETURN_ERROR_IF(!repCode && offCode - ZSTD_REP_MOVE > offsetBound, corruption_detected, "Offset too large!"); RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!");
RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small"); RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
return 0; return 0;
} }
#endif
/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32* const rep, U32 ll0) { static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32* const rep, U32 ll0) {
@ -4539,7 +4537,7 @@ static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_
const void* src, size_t blockSize, ZSTD_CCtx* cctx) { const void* src, size_t blockSize, ZSTD_CCtx* cctx) {
size_t idx = seqPos->idx; size_t idx = seqPos->idx;
BYTE const* ip = (BYTE const*)(src); BYTE const* ip = (BYTE const*)(src);
const BYTE const* iend = ip + blockSize; const BYTE* const iend = ip + blockSize;
repcodes_t updatedRepcodes; repcodes_t updatedRepcodes;
U32 dictSize; U32 dictSize;
U32 litLength; U32 litLength;
@ -4563,14 +4561,11 @@ static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_
updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
seqPos->posInSrc += litLength; seqPos->posInSrc += litLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, repCode, matchLength, FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
seqPos->posInSrc, cctx->appliedParams.cParams.windowLog, cctx->appliedParams.cParams.windowLog, dictSize),
dictSize),
"Sequence validation failed"); "Sequence validation failed");
seqPos->posInSrc += matchLength; seqPos->posInSrc += matchLength;
#endif
ZSTD_storeSeq(seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); ZSTD_storeSeq(seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH);
ip += matchLength + litLength; ip += matchLength + litLength;
} }
@ -4580,9 +4575,7 @@ static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_
DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
ZSTD_storeLastLiterals(seqStore, ip, inSeqs[idx].litLength); ZSTD_storeLastLiterals(seqStore, ip, inSeqs[idx].litLength);
ip += inSeqs[idx].litLength; ip += inSeqs[idx].litLength;
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
seqPos->posInSrc += inSeqs[idx].litLength; seqPos->posInSrc += inSeqs[idx].litLength;
#endif
} }
RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!");
seqPos->idx = idx+1; seqPos->idx = idx+1;
@ -4616,7 +4609,6 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
U32 matchLength; U32 matchLength;
U32 rawOffset; U32 rawOffset;
U32 offCode; U32 offCode;
U32 repCode;
if (cctx->cdict) { if (cctx->cdict) {
dictSize = cctx->cdict->dictContentSize; dictSize = cctx->cdict->dictContentSize;
@ -4625,15 +4617,14 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
} else { } else {
dictSize = 0; dictSize = 0;
} }
DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %zu PIS: %u blockSize: %zu windowSize: %u", idx, startPosInSequence, blockSize, windowSize); DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %zu PIS: %zu blockSize: %zu", idx, startPosInSequence, blockSize);
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); DEBUGLOG(5, "Start seq: idx: %zu (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
const ZSTD_Sequence currSeq = inSeqs[idx]; const ZSTD_Sequence currSeq = inSeqs[idx];
litLength = currSeq.litLength; litLength = currSeq.litLength;
matchLength = currSeq.matchLength; matchLength = currSeq.matchLength;
rawOffset = currSeq.offset; rawOffset = currSeq.offset;
repCode = 0;
/* Modify the sequence depending on where endPosInSequence lies */ /* Modify the sequence depending on where endPosInSequence lies */
if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
@ -4651,11 +4642,12 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
} else { } else {
/* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
does not reach the end of the match. So, we have to split the sequence */ does not reach the end of the match. So, we have to split the sequence */
DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", DEBUGLOG(6, "Require a split: diff: %zu, idx: %zu PIS: %zu",
currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
if (endPosInSequence > litLength) { if (endPosInSequence > litLength) {
U32 firstHalfMatchLength;
litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
U32 firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
if (matchLength > blockSize && firstHalfMatchLength >= MINMATCH) { if (matchLength > blockSize && firstHalfMatchLength >= MINMATCH) {
/* Only ever split the match if it is larger than the block size */ /* Only ever split the match if it is larger than the block size */
U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence;
@ -4690,14 +4682,11 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
} }
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
seqPos->posInSrc += litLength; seqPos->posInSrc += litLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, repCode, FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
matchLength, seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize), cctx->appliedParams.cParams.windowLog, dictSize),
"Sequence validation failed"); "Sequence validation failed");
seqPos->posInSrc += matchLength; seqPos->posInSrc += matchLength;
#endif
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
ZSTD_storeSeq(seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH); ZSTD_storeSeq(seqStore, litLength, ip, iend, offCode, matchLength - MINMATCH);
ip += matchLength + litLength; ip += matchLength + litLength;
@ -4714,9 +4703,7 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
assert(ip <= iend); assert(ip <= iend);
DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
ZSTD_storeLastLiterals(seqStore, ip, lastLLSize); ZSTD_storeLastLiterals(seqStore, ip, lastLLSize);
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
seqPos->posInSrc += lastLLSize; seqPos->posInSrc += lastLLSize;
#endif
} }
return bytesAdjustment; return bytesAdjustment;