facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[test] Add large dict/data --patch-from test 

Dictionary size must be > `ZSTD_CHUNKSIZE_MAX`.
dev
Nick Terrell 2021-05-04 17:09:32 -07:00
parent 94db4398a0
commit 0b88c2582c
2 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lib/compress/zstd_compress.c
View File

@ -4121,6 +4121,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
/* Assert that we the ms params match the params we're being given */ /* Assert that we the ms params match the params we're being given */
ZSTD_assertEqualCParams(params->cParams, ms->cParams); ZSTD_assertEqualCParams(params->cParams, ms->cParams);
if (srcSize > ZSTD_CHUNKSIZE_MAX) { if (srcSize > ZSTD_CHUNKSIZE_MAX) {
/* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
* Dictionaries right at the edge will immediately trigger overflow * Dictionaries right at the edge will immediately trigger overflow
@ -4153,7 +4154,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
if (loadLdmDict) if (loadLdmDict)
ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, &params->ldmParams); ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
switch(params->cParams.strategy) switch(params->cParams.strategy)
{ {
@ -4167,22 +4168,20 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_greedy: case ZSTD_greedy:
case ZSTD_lazy: case ZSTD_lazy:
case ZSTD_lazy2: case ZSTD_lazy2:
if (srcSize >= HASH_READ_SIZE) { assert(srcSize >= HASH_READ_SIZE);
if (ms->dedicatedDictSearch) { if (ms->dedicatedDictSearch) {
assert(ms->chainTable != NULL); assert(ms->chainTable != NULL);
ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
} else {
assert(params->useRowMatchFinder != ZSTD_urm_auto);
if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) {
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
} else { } else {
assert(params->useRowMatchFinder != ZSTD_urm_auto); ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) { DEBUGLOG(4, "Using chain-based hash table for lazy dict");
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
if (ip == src)
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
} else {
ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using chain-based hash table for lazy dict");
}
} }
} }
break; break;
@ -4191,8 +4190,8 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btultra: case ZSTD_btultra:
case ZSTD_btultra2: case ZSTD_btultra2:
if (srcSize >= HASH_READ_SIZE) assert(srcSize >= HASH_READ_SIZE);
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
break; break;
default: default:

8
tests/playTests.sh
View File

@ -1430,6 +1430,14 @@ datagen -g5000000 > tmp_patch
zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered"
rm -rf tmp* rm -rf tmp*
println "\n===> patch-from very large dictionary and file test"
datagen -g550000000 -P0 > tmp_dict
datagen -g100000000 -P1 > tmp_patch
zstd --long=30 -1f --patch-from tmp_dict tmp_patch
zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp*
println "\n===> patch-from --stream-size test" println "\n===> patch-from --stream-size test"
datagen -g1000 -P50 > tmp_dict datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch datagen -g1000 -P10 > tmp_patch