facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[test] Add large dict/data --patch-from test 

Dictionary size must be > `ZSTD_CHUNKSIZE_MAX`.
dev
Nick Terrell 2021-05-04 17:09:32 -07:00
parent 94db4398a0
commit 0b88c2582c
2 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lib/compress/zstd_compress.c
View File

@ -4121,6 +4121,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
/* Assert that we the ms params match the params we're being given */
ZSTD_assertEqualCParams(params->cParams, ms->cParams);
if (srcSize > ZSTD_CHUNKSIZE_MAX) {
/* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
* Dictionaries right at the edge will immediately trigger overflow
@ -4153,7 +4154,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
if (loadLdmDict)
ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, &params->ldmParams);
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
switch(params->cParams.strategy)
{
@ -4167,22 +4168,20 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_greedy:
case ZSTD_lazy:
case ZSTD_lazy2:
if (srcSize >= HASH_READ_SIZE) {
if (ms->dedicatedDictSearch) {
assert(ms->chainTable != NULL);
ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
assert(srcSize >= HASH_READ_SIZE);
if (ms->dedicatedDictSearch) {
assert(ms->chainTable != NULL);
ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
} else {
assert(params->useRowMatchFinder != ZSTD_urm_auto);
if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) {
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
} else {
assert(params->useRowMatchFinder != ZSTD_urm_auto);
if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) {
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
if (ip == src)
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
} else {
ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using chain-based hash table for lazy dict");
}
ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using chain-based hash table for lazy dict");
}
}
break;
@ -4191,8 +4190,8 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
if (srcSize >= HASH_READ_SIZE)
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
assert(srcSize >= HASH_READ_SIZE);
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
break;
default:

8
tests/playTests.sh
View File

@ -1430,6 +1430,14 @@ datagen -g5000000 > tmp_patch
zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered"
rm -rf tmp*
println "\n===> patch-from very large dictionary and file test"
datagen -g550000000 -P0 > tmp_dict
datagen -g100000000 -P1 > tmp_patch
zstd --long=30 -1f --patch-from tmp_dict tmp_patch
zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp*
println "\n===> patch-from --stream-size test"
datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch