facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

btlazy2 : optimization for dictionary compression

Browse Source 
we want the dictionary table to be fully sorted,
not just lazily filled.
Dictionary loading is a bit more intensive,
but it saves cpu cycles for match search during compression.
This commit is contained in:
Yann Collet 2017-12-29 19:14:18 +01:00
parent 02f64ef955
commit d228b6b0d0
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/compress/zstd_compress.c
View File

@ -1950,11 +1950,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength); ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength);
break; break;
case ZSTD_btlazy2: case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
if (srcSize >= HASH_READ_SIZE)
ZSTD_updateDUBT(zc, iend-HASH_READ_SIZE, iend, zc->appliedParams.cParams.searchLength);
break;
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btultra: case ZSTD_btultra:
if (srcSize >= HASH_READ_SIZE) if (srcSize >= HASH_READ_SIZE)

5
tests/playTests.sh
View File

@ -322,9 +322,14 @@ $ECHO "- Create first dictionary "
TESTFILE=../programs/zstdcli.c TESTFILE=../programs/zstdcli.c
$ZSTD --train *.c ../programs/*.c -o tmpDict $ZSTD --train *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp cp $TESTFILE tmp
$ECHO "- Dictionary compression roundtrip"
$ZSTD -f tmp -D tmpDict $ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result $ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result $DIFF $TESTFILE result
$ECHO "- Dictionary compression with btlazy2 strategy"
$ZSTD -f tmp -D tmpDict --zstd=strategy=6
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
if [ -n "$hasMT" ] if [ -n "$hasMT" ]
then then
$ECHO "- Test dictionary compression with multithreading " $ECHO "- Test dictionary compression with multithreading "