facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2820 from terrelln/nb-compares

Browse Source 
[binary-tree] Fix underflow of nbCompares
This commit is contained in:
Nick Terrell 2021-10-11 09:59:57 -07:00 committed by GitHub
commit b77d95b053
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/compress/zstd_lazy.c
View File

@ -93,7 +93,7 @@ ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
assert(curr >= btLow); assert(curr >= btLow);
assert(ip < iend); /* condition for ZSTD_count */ assert(ip < iend); /* condition for ZSTD_count */
while (nbCompares-- && (matchIndex > windowLow)) { for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < curr); assert(matchIndex < curr);
@ -185,7 +185,7 @@ ZSTD_DUBT_findBetterDictMatch (
(void)dictMode; (void)dictMode;
assert(dictMode == ZSTD_dictMatchState); assert(dictMode == ZSTD_dictMatchState);
while (nbCompares-- && (dictMatchIndex > dictLowLimit)) { for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) {
U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match = dictBase + dictMatchIndex; const BYTE* match = dictBase + dictMatchIndex;
@ -309,7 +309,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
matchIndex = hashTable[h]; matchIndex = hashTable[h];
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) { for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match; const BYTE* match;
@ -357,6 +357,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dictMatchState && nbCompares) { if (dictMode == ZSTD_dictMatchState && nbCompares) {
bestLength = ZSTD_DUBT_findBetterDictMatch( bestLength = ZSTD_DUBT_findBetterDictMatch(
ms, ip, iend, ms, ip, iend,
@ -758,6 +759,7 @@ size_t ZSTD_HcFindBestMatch_generic (
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
} }
assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dedicatedDictSearch) { if (dictMode == ZSTD_dedicatedDictSearch) {
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
@ -1356,6 +1358,7 @@ size_t ZSTD_RowFindBestMatch_generic (
} }
} }
assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dedicatedDictSearch) { if (dictMode == ZSTD_dedicatedDictSearch) {
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);

7
lib/compress/zstd_opt.c
View File

@ -442,7 +442,7 @@ static U32 ZSTD_insertBt1(
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
assert(windowLow > 0); assert(windowLow > 0);
while (nbCompares-- && (matchIndex >= windowLow)) { for (; nbCompares && (matchIndex >= windowLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < curr); assert(matchIndex < curr);
@ -673,7 +673,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
hashTable[h] = curr; /* Update Hash Table */ hashTable[h] = curr; /* Update Hash Table */
while (nbCompares-- && (matchIndex >= matchLow)) { for (; nbCompares && (matchIndex >= matchLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask); U32* const nextPtr = bt + 2*(matchIndex & btMask);
const BYTE* match; const BYTE* match;
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
@ -725,12 +725,13 @@ U32 ZSTD_insertBtAndGetAllMatches (
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dictMatchState && nbCompares) { if (dictMode == ZSTD_dictMatchState && nbCompares) {
size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls);
U32 dictMatchIndex = dms->hashTable[dmsH]; U32 dictMatchIndex = dms->hashTable[dmsH];
const U32* const dmsBt = dms->chainTable; const U32* const dmsBt = dms->chainTable;
commonLengthSmaller = commonLengthLarger = 0; commonLengthSmaller = commonLengthLarger = 0;
while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) {
const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match = dmsBase + dictMatchIndex; const BYTE* match = dmsBase + dictMatchIndex;

40
tests/regression/results.csv
View File

@ -171,7 +171,7 @@ github, level 7 with dict, zstdcli,
github, level 9, zstdcli, 137122 github, level 9, zstdcli, 137122
github, level 9 with dict, zstdcli, 41393 github, level 9 with dict, zstdcli, 41393
github, level 13, zstdcli, 136064 github, level 13, zstdcli, 136064
github, level 13 with dict, zstdcli, 41743 github, level 13 with dict, zstdcli, 41900
github, level 16, zstdcli, 136064 github, level 16, zstdcli, 136064
github, level 16 with dict, zstdcli, 39577 github, level 16 with dict, zstdcli, 39577
github, level 19, zstdcli, 136064 github, level 19, zstdcli, 136064
@ -391,9 +391,9 @@ github, level 12 row 2 with dict dds, advanced
github, level 12 row 2 with dict copy, advanced one pass, 39677 github, level 12 row 2 with dict copy, advanced one pass, 39677
github, level 12 row 2 with dict load, advanced one pass, 41166 github, level 12 row 2 with dict load, advanced one pass, 41166
github, level 13, advanced one pass, 134064 github, level 13, advanced one pass, 134064
github, level 13 with dict, advanced one pass, 39743 github, level 13 with dict, advanced one pass, 39900
github, level 13 with dict dms, advanced one pass, 39743 github, level 13 with dict dms, advanced one pass, 39900
github, level 13 with dict dds, advanced one pass, 39743 github, level 13 with dict dds, advanced one pass, 39900
github, level 13 with dict copy, advanced one pass, 39948 github, level 13 with dict copy, advanced one pass, 39948
github, level 13 with dict load, advanced one pass, 42626 github, level 13 with dict load, advanced one pass, 42626
github, level 16, advanced one pass, 134064 github, level 16, advanced one pass, 134064
@ -517,8 +517,8 @@ github.tar, level 12 row 2 with dict copy, advanced
github.tar, level 12 row 2 with dict load, advanced one pass, 36459 github.tar, level 12 row 2 with dict load, advanced one pass, 36459
github.tar, level 13, advanced one pass, 35501 github.tar, level 13, advanced one pass, 35501
github.tar, level 13 with dict, advanced one pass, 37130 github.tar, level 13 with dict, advanced one pass, 37130
github.tar, level 13 with dict dms, advanced one pass, 37267 github.tar, level 13 with dict dms, advanced one pass, 37220
github.tar, level 13 with dict dds, advanced one pass, 37267 github.tar, level 13 with dict dds, advanced one pass, 37220
github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict copy, advanced one pass, 37130
github.tar, level 13 with dict load, advanced one pass, 36010 github.tar, level 13 with dict load, advanced one pass, 36010
github.tar, level 16, advanced one pass, 40471 github.tar, level 16, advanced one pass, 40471
@ -709,9 +709,9 @@ github, level 12 row 2 with dict dds, advanced
github, level 12 row 2 with dict copy, advanced one pass small out, 39677 github, level 12 row 2 with dict copy, advanced one pass small out, 39677
github, level 12 row 2 with dict load, advanced one pass small out, 41166 github, level 12 row 2 with dict load, advanced one pass small out, 41166
github, level 13, advanced one pass small out, 134064 github, level 13, advanced one pass small out, 134064
github, level 13 with dict, advanced one pass small out, 39743 github, level 13 with dict, advanced one pass small out, 39900
github, level 13 with dict dms, advanced one pass small out, 39743 github, level 13 with dict dms, advanced one pass small out, 39900
github, level 13 with dict dds, advanced one pass small out, 39743 github, level 13 with dict dds, advanced one pass small out, 39900
github, level 13 with dict copy, advanced one pass small out, 39948 github, level 13 with dict copy, advanced one pass small out, 39948
github, level 13 with dict load, advanced one pass small out, 42626 github, level 13 with dict load, advanced one pass small out, 42626
github, level 16, advanced one pass small out, 134064 github, level 16, advanced one pass small out, 134064
@ -835,8 +835,8 @@ github.tar, level 12 row 2 with dict copy, advanced
github.tar, level 12 row 2 with dict load, advanced one pass small out, 36459 github.tar, level 12 row 2 with dict load, advanced one pass small out, 36459
github.tar, level 13, advanced one pass small out, 35501 github.tar, level 13, advanced one pass small out, 35501
github.tar, level 13 with dict, advanced one pass small out, 37130 github.tar, level 13 with dict, advanced one pass small out, 37130
github.tar, level 13 with dict dms, advanced one pass small out, 37267 github.tar, level 13 with dict dms, advanced one pass small out, 37220
github.tar, level 13 with dict dds, advanced one pass small out, 37267 github.tar, level 13 with dict dds, advanced one pass small out, 37220
github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict copy, advanced one pass small out, 37130
github.tar, level 13 with dict load, advanced one pass small out, 36010 github.tar, level 13 with dict load, advanced one pass small out, 36010
github.tar, level 16, advanced one pass small out, 40471 github.tar, level 16, advanced one pass small out, 40471
@ -1027,9 +1027,9 @@ github, level 12 row 2 with dict dds, advanced
github, level 12 row 2 with dict copy, advanced streaming, 39677 github, level 12 row 2 with dict copy, advanced streaming, 39677
github, level 12 row 2 with dict load, advanced streaming, 41166 github, level 12 row 2 with dict load, advanced streaming, 41166
github, level 13, advanced streaming, 134064 github, level 13, advanced streaming, 134064
github, level 13 with dict, advanced streaming, 39743 github, level 13 with dict, advanced streaming, 39900
github, level 13 with dict dms, advanced streaming, 39743 github, level 13 with dict dms, advanced streaming, 39900
github, level 13 with dict dds, advanced streaming, 39743 github, level 13 with dict dds, advanced streaming, 39900
github, level 13 with dict copy, advanced streaming, 39948 github, level 13 with dict copy, advanced streaming, 39948
github, level 13 with dict load, advanced streaming, 42626 github, level 13 with dict load, advanced streaming, 42626
github, level 16, advanced streaming, 134064 github, level 16, advanced streaming, 134064
@ -1153,8 +1153,8 @@ github.tar, level 12 row 2 with dict copy, advanced
github.tar, level 12 row 2 with dict load, advanced streaming, 36459 github.tar, level 12 row 2 with dict load, advanced streaming, 36459
github.tar, level 13, advanced streaming, 35501 github.tar, level 13, advanced streaming, 35501
github.tar, level 13 with dict, advanced streaming, 37130 github.tar, level 13 with dict, advanced streaming, 37130
github.tar, level 13 with dict dms, advanced streaming, 37267 github.tar, level 13 with dict dms, advanced streaming, 37220
github.tar, level 13 with dict dds, advanced streaming, 37267 github.tar, level 13 with dict dds, advanced streaming, 37220
github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict copy, advanced streaming, 37130
github.tar, level 13 with dict load, advanced streaming, 36010 github.tar, level 13 with dict load, advanced streaming, 36010
github.tar, level 16, advanced streaming, 40471 github.tar, level 16, advanced streaming, 40471
@ -1241,7 +1241,7 @@ github, level 7 with dict, old stre
github, level 9, old streaming, 135122 github, level 9, old streaming, 135122
github, level 9 with dict, old streaming, 39437 github, level 9 with dict, old streaming, 39437
github, level 13, old streaming, 134064 github, level 13, old streaming, 134064
github, level 13 with dict, old streaming, 39743 github, level 13 with dict, old streaming, 39900
github, level 16, old streaming, 134064 github, level 16, old streaming, 134064
github, level 16 with dict, old streaming, 37577 github, level 16 with dict, old streaming, 37577
github, level 19, old streaming, 134064 github, level 19, old streaming, 134064
@ -1359,7 +1359,7 @@ github, level 7 with dict, old stre
github, level 9, old streaming advanced, 138676 github, level 9, old streaming advanced, 138676
github, level 9 with dict, old streaming advanced, 38981 github, level 9 with dict, old streaming advanced, 38981
github, level 13, old streaming advanced, 138676 github, level 13, old streaming advanced, 138676
github, level 13 with dict, old streaming advanced, 39721 github, level 13 with dict, old streaming advanced, 39725
github, level 16, old streaming advanced, 138676 github, level 16, old streaming advanced, 138676
github, level 16 with dict, old streaming advanced, 40789 github, level 16 with dict, old streaming advanced, 40789
github, level 19, old streaming advanced, 134064 github, level 19, old streaming advanced, 134064
@ -1429,7 +1429,7 @@ github, level 5 with dict, old stre
github, level 6 with dict, old streaming cdict, 38671 github, level 6 with dict, old streaming cdict, 38671
github, level 7 with dict, old streaming cdict, 38758 github, level 7 with dict, old streaming cdict, 38758
github, level 9 with dict, old streaming cdict, 39437 github, level 9 with dict, old streaming cdict, 39437
github, level 13 with dict, old streaming cdict, 39743 github, level 13 with dict, old streaming cdict, 39900
github, level 16 with dict, old streaming cdict, 37577 github, level 16 with dict, old streaming cdict, 37577
github, level 19 with dict, old streaming cdict, 37576 github, level 19 with dict, old streaming cdict, 37576
github, no source size with dict, old streaming cdict, 40654 github, no source size with dict, old streaming cdict, 40654
@ -1459,7 +1459,7 @@ github, level 5 with dict, old stre
github, level 6 with dict, old streaming advanced cdict, 39363 github, level 6 with dict, old streaming advanced cdict, 39363
github, level 7 with dict, old streaming advanced cdict, 38924 github, level 7 with dict, old streaming advanced cdict, 38924
github, level 9 with dict, old streaming advanced cdict, 38981 github, level 9 with dict, old streaming advanced cdict, 38981
github, level 13 with dict, old streaming advanced cdict, 39721 github, level 13 with dict, old streaming advanced cdict, 39725
github, level 16 with dict, old streaming advanced cdict, 40789 github, level 16 with dict, old streaming advanced cdict, 40789
github, level 19 with dict, old streaming advanced cdict, 37576 github, level 19 with dict, old streaming advanced cdict, 37576
github, no source size with dict, old streaming advanced cdict, 40608 github, no source size with dict, old streaming advanced cdict, 40608

1 Data Config Method Total compressed size
171 github level 9 zstdcli 137122
172 github level 9 with dict zstdcli 41393
173 github level 13 zstdcli 136064
174 github level 13 with dict zstdcli 41743 41900
175 github level 16 zstdcli 136064
176 github level 16 with dict zstdcli 39577
177 github level 19 zstdcli 136064
391 github level 12 row 2 with dict copy advanced one pass 39677
392 github level 12 row 2 with dict load advanced one pass 41166
393 github level 13 advanced one pass 134064
394 github level 13 with dict advanced one pass 39743 39900
395 github level 13 with dict dms advanced one pass 39743 39900
396 github level 13 with dict dds advanced one pass 39743 39900
397 github level 13 with dict copy advanced one pass 39948
398 github level 13 with dict load advanced one pass 42626
399 github level 16 advanced one pass 134064
517 github.tar level 12 row 2 with dict load advanced one pass 36459
518 github.tar level 13 advanced one pass 35501
519 github.tar level 13 with dict advanced one pass 37130
520 github.tar level 13 with dict dms advanced one pass 37267 37220
521 github.tar level 13 with dict dds advanced one pass 37267 37220
522 github.tar level 13 with dict copy advanced one pass 37130
523 github.tar level 13 with dict load advanced one pass 36010
524 github.tar level 16 advanced one pass 40471
709 github level 12 row 2 with dict copy advanced one pass small out 39677
710 github level 12 row 2 with dict load advanced one pass small out 41166
711 github level 13 advanced one pass small out 134064
712 github level 13 with dict advanced one pass small out 39743 39900
713 github level 13 with dict dms advanced one pass small out 39743 39900
714 github level 13 with dict dds advanced one pass small out 39743 39900
715 github level 13 with dict copy advanced one pass small out 39948
716 github level 13 with dict load advanced one pass small out 42626
717 github level 16 advanced one pass small out 134064
835 github.tar level 12 row 2 with dict load advanced one pass small out 36459
836 github.tar level 13 advanced one pass small out 35501
837 github.tar level 13 with dict advanced one pass small out 37130
838 github.tar level 13 with dict dms advanced one pass small out 37267 37220
839 github.tar level 13 with dict dds advanced one pass small out 37267 37220
840 github.tar level 13 with dict copy advanced one pass small out 37130
841 github.tar level 13 with dict load advanced one pass small out 36010
842 github.tar level 16 advanced one pass small out 40471
1027 github level 12 row 2 with dict copy advanced streaming 39677
1028 github level 12 row 2 with dict load advanced streaming 41166
1029 github level 13 advanced streaming 134064
1030 github level 13 with dict advanced streaming 39743 39900
1031 github level 13 with dict dms advanced streaming 39743 39900
1032 github level 13 with dict dds advanced streaming 39743 39900
1033 github level 13 with dict copy advanced streaming 39948
1034 github level 13 with dict load advanced streaming 42626
1035 github level 16 advanced streaming 134064
1153 github.tar level 12 row 2 with dict load advanced streaming 36459
1154 github.tar level 13 advanced streaming 35501
1155 github.tar level 13 with dict advanced streaming 37130
1156 github.tar level 13 with dict dms advanced streaming 37267 37220
1157 github.tar level 13 with dict dds advanced streaming 37267 37220
1158 github.tar level 13 with dict copy advanced streaming 37130
1159 github.tar level 13 with dict load advanced streaming 36010
1160 github.tar level 16 advanced streaming 40471
1241 github level 9 old streaming 135122
1242 github level 9 with dict old streaming 39437
1243 github level 13 old streaming 134064
1244 github level 13 with dict old streaming 39743 39900
1245 github level 16 old streaming 134064
1246 github level 16 with dict old streaming 37577
1247 github level 19 old streaming 134064
1359 github level 9 old streaming advanced 138676
1360 github level 9 with dict old streaming advanced 38981
1361 github level 13 old streaming advanced 138676
1362 github level 13 with dict old streaming advanced 39721 39725
1363 github level 16 old streaming advanced 138676
1364 github level 16 with dict old streaming advanced 40789
1365 github level 19 old streaming advanced 134064
1429 github level 6 with dict old streaming cdict 38671
1430 github level 7 with dict old streaming cdict 38758
1431 github level 9 with dict old streaming cdict 39437
1432 github level 13 with dict old streaming cdict 39743 39900
1433 github level 16 with dict old streaming cdict 37577
1434 github level 19 with dict old streaming cdict 37576
1435 github no source size with dict old streaming cdict 40654
1459 github level 6 with dict old streaming advanced cdict 39363
1460 github level 7 with dict old streaming advanced cdict 38924
1461 github level 9 with dict old streaming advanced cdict 38981
1462 github level 13 with dict old streaming advanced cdict 39721 39725
1463 github level 16 with dict old streaming advanced cdict 40789
1464 github level 19 with dict old streaming advanced cdict 37576
1465 github no source size with dict old streaming advanced cdict 40608