facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2983 from facebook/minLitPricev2

Browse Source 
[opt] minor compression ratio improvement
This commit is contained in:
Yann Collet 2022-01-20 16:02:31 -08:00 committed by GitHub
commit 71921e596f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/compress/zstd_opt.c
View File

@ -255,11 +255,14 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */
/* dynamic statistics */ /* dynamic statistics */
{ U32 price = litLength * optPtr->litSumBasePrice; { U32 price = optPtr->litSumBasePrice * litLength;
U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
U32 u; U32 u;
assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
for (u=0; u < litLength; u++) { for (u=0; u < litLength; u++) {
assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
price -= litPrice;
} }
return price; return price;
} }

20
tests/regression/results.csv
View File

@ -46,7 +46,7 @@ silesia, level 7, compress
silesia, level 9, compress cctx, 4543018 silesia, level 9, compress cctx, 4543018
silesia, level 13, compress cctx, 4493990 silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4359864 silesia, level 16, compress cctx, 4359864
silesia, level 19, compress cctx, 4296880 silesia, level 19, compress cctx, 4296686
silesia, long distance mode, compress cctx, 4842075 silesia, long distance mode, compress cctx, 4842075
silesia, multithreaded, compress cctx, 4842075 silesia, multithreaded, compress cctx, 4842075
silesia, multithreaded long distance mode, compress cctx, 4842075 silesia, multithreaded long distance mode, compress cctx, 4842075
@ -55,7 +55,7 @@ silesia, small hash log, compress
silesia, small chain log, compress cctx, 4912197 silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794052 silesia, explicit params, compress cctx, 4794052
silesia, uncompressed literals, compress cctx, 4842075 silesia, uncompressed literals, compress cctx, 4842075
silesia, uncompressed literals optimal, compress cctx, 4296880 silesia, uncompressed literals optimal, compress cctx, 4296686
silesia, huffman literals, compress cctx, 6172178 silesia, huffman literals, compress cctx, 6172178
silesia, multithreaded with advanced params, compress cctx, 4842075 silesia, multithreaded with advanced params, compress cctx, 4842075
github, level -5, compress cctx, 204411 github, level -5, compress cctx, 204411
@ -110,7 +110,7 @@ silesia, level 7, zstdcli,
silesia, level 9, zstdcli, 4543066 silesia, level 9, zstdcli, 4543066
silesia, level 13, zstdcli, 4494038 silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4359912 silesia, level 16, zstdcli, 4359912
silesia, level 19, zstdcli, 4296928 silesia, level 19, zstdcli, 4296734
silesia, long distance mode, zstdcli, 4833785 silesia, long distance mode, zstdcli, 4833785
silesia, multithreaded, zstdcli, 4842123 silesia, multithreaded, zstdcli, 4842123
silesia, multithreaded long distance mode, zstdcli, 4833785 silesia, multithreaded long distance mode, zstdcli, 4833785
@ -249,7 +249,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced one pass, 4503116 silesia, level 12 row 2, advanced one pass, 4503116
silesia, level 13, advanced one pass, 4493990 silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4359864 silesia, level 16, advanced one pass, 4359864
silesia, level 19, advanced one pass, 4296880 silesia, level 19, advanced one pass, 4296686
silesia, no source size, advanced one pass, 4842075 silesia, no source size, advanced one pass, 4842075
silesia, long distance mode, advanced one pass, 4833710 silesia, long distance mode, advanced one pass, 4833710
silesia, multithreaded, advanced one pass, 4842075 silesia, multithreaded, advanced one pass, 4842075
@ -567,7 +567,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced one pass small out, 4503116 silesia, level 12 row 2, advanced one pass small out, 4503116
silesia, level 13, advanced one pass small out, 4493990 silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4359864 silesia, level 16, advanced one pass small out, 4359864
silesia, level 19, advanced one pass small out, 4296880 silesia, level 19, advanced one pass small out, 4296686
silesia, no source size, advanced one pass small out, 4842075 silesia, no source size, advanced one pass small out, 4842075
silesia, long distance mode, advanced one pass small out, 4833710 silesia, long distance mode, advanced one pass small out, 4833710
silesia, multithreaded, advanced one pass small out, 4842075 silesia, multithreaded, advanced one pass small out, 4842075
@ -885,7 +885,7 @@ silesia, level 12 row 1, advanced
silesia, level 12 row 2, advanced streaming, 4503116 silesia, level 12 row 2, advanced streaming, 4503116
silesia, level 13, advanced streaming, 4493990 silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4359864 silesia, level 16, advanced streaming, 4359864
silesia, level 19, advanced streaming, 4296880 silesia, level 19, advanced streaming, 4296686
silesia, no source size, advanced streaming, 4842039 silesia, no source size, advanced streaming, 4842039
silesia, long distance mode, advanced streaming, 4833710 silesia, long distance mode, advanced streaming, 4833710
silesia, multithreaded, advanced streaming, 4842075 silesia, multithreaded, advanced streaming, 4842075
@ -1195,10 +1195,10 @@ silesia, level 7, old stre
silesia, level 9, old streaming, 4543018 silesia, level 9, old streaming, 4543018
silesia, level 13, old streaming, 4493990 silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4359864 silesia, level 16, old streaming, 4359864
silesia, level 19, old streaming, 4296880 silesia, level 19, old streaming, 4296686
silesia, no source size, old streaming, 4842039 silesia, no source size, old streaming, 4842039
silesia, uncompressed literals, old streaming, 4842075 silesia, uncompressed literals, old streaming, 4842075
silesia, uncompressed literals optimal, old streaming, 4296880 silesia, uncompressed literals optimal, old streaming, 4296686
silesia, huffman literals, old streaming, 6179294 silesia, huffman literals, old streaming, 6179294
silesia.tar, level -5, old streaming, 7043687 silesia.tar, level -5, old streaming, 7043687
silesia.tar, level -3, old streaming, 6671317 silesia.tar, level -3, old streaming, 6671317
@ -1297,7 +1297,7 @@ silesia, level 7, old stre
silesia, level 9, old streaming advanced, 4543018 silesia, level 9, old streaming advanced, 4543018
silesia, level 13, old streaming advanced, 4493990 silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4359864 silesia, level 16, old streaming advanced, 4359864
silesia, level 19, old streaming advanced, 4296880 silesia, level 19, old streaming advanced, 4296686
silesia, no source size, old streaming advanced, 4842039 silesia, no source size, old streaming advanced, 4842039
silesia, long distance mode, old streaming advanced, 4842075 silesia, long distance mode, old streaming advanced, 4842075
silesia, multithreaded, old streaming advanced, 4842075 silesia, multithreaded, old streaming advanced, 4842075
@ -1307,7 +1307,7 @@ silesia, small hash log, old stre
silesia, small chain log, old streaming advanced, 4912197 silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795452 silesia, explicit params, old streaming advanced, 4795452
silesia, uncompressed literals, old streaming advanced, 4842075 silesia, uncompressed literals, old streaming advanced, 4842075
silesia, uncompressed literals optimal, old streaming advanced, 4296880 silesia, uncompressed literals optimal, old streaming advanced, 4296686
silesia, huffman literals, old streaming advanced, 6179294 silesia, huffman literals, old streaming advanced, 6179294
silesia, multithreaded with advanced params, old streaming advanced, 4842075 silesia, multithreaded with advanced params, old streaming advanced, 4842075
silesia.tar, level -5, old streaming advanced, 7043687 silesia.tar, level -5, old streaming advanced, 7043687

1 Data Config Method Total compressed size
46 silesia level 9 compress cctx 4543018
47 silesia level 13 compress cctx 4493990
48 silesia level 16 compress cctx 4359864
49 silesia level 19 compress cctx 4296880 4296686
50 silesia long distance mode compress cctx 4842075
51 silesia multithreaded compress cctx 4842075
52 silesia multithreaded long distance mode compress cctx 4842075
55 silesia small chain log compress cctx 4912197
56 silesia explicit params compress cctx 4794052
57 silesia uncompressed literals compress cctx 4842075
58 silesia uncompressed literals optimal compress cctx 4296880 4296686
59 silesia huffman literals compress cctx 6172178
60 silesia multithreaded with advanced params compress cctx 4842075
61 github level -5 compress cctx 204411
110 silesia level 9 zstdcli 4543066
111 silesia level 13 zstdcli 4494038
112 silesia level 16 zstdcli 4359912
113 silesia level 19 zstdcli 4296928 4296734
114 silesia long distance mode zstdcli 4833785
115 silesia multithreaded zstdcli 4842123
116 silesia multithreaded long distance mode zstdcli 4833785
249 silesia level 12 row 2 advanced one pass 4503116
250 silesia level 13 advanced one pass 4493990
251 silesia level 16 advanced one pass 4359864
252 silesia level 19 advanced one pass 4296880 4296686
253 silesia no source size advanced one pass 4842075
254 silesia long distance mode advanced one pass 4833710
255 silesia multithreaded advanced one pass 4842075
567 silesia level 12 row 2 advanced one pass small out 4503116
568 silesia level 13 advanced one pass small out 4493990
569 silesia level 16 advanced one pass small out 4359864
570 silesia level 19 advanced one pass small out 4296880 4296686
571 silesia no source size advanced one pass small out 4842075
572 silesia long distance mode advanced one pass small out 4833710
573 silesia multithreaded advanced one pass small out 4842075
885 silesia level 12 row 2 advanced streaming 4503116
886 silesia level 13 advanced streaming 4493990
887 silesia level 16 advanced streaming 4359864
888 silesia level 19 advanced streaming 4296880 4296686
889 silesia no source size advanced streaming 4842039
890 silesia long distance mode advanced streaming 4833710
891 silesia multithreaded advanced streaming 4842075
1195 silesia level 9 old streaming 4543018
1196 silesia level 13 old streaming 4493990
1197 silesia level 16 old streaming 4359864
1198 silesia level 19 old streaming 4296880 4296686
1199 silesia no source size old streaming 4842039
1200 silesia uncompressed literals old streaming 4842075
1201 silesia uncompressed literals optimal old streaming 4296880 4296686
1202 silesia huffman literals old streaming 6179294
1203 silesia.tar level -5 old streaming 7043687
1204 silesia.tar level -3 old streaming 6671317
1297 silesia level 9 old streaming advanced 4543018
1298 silesia level 13 old streaming advanced 4493990
1299 silesia level 16 old streaming advanced 4359864
1300 silesia level 19 old streaming advanced 4296880 4296686
1301 silesia no source size old streaming advanced 4842039
1302 silesia long distance mode old streaming advanced 4842075
1303 silesia multithreaded old streaming advanced 4842075
1307 silesia small chain log old streaming advanced 4912197
1308 silesia explicit params old streaming advanced 4795452
1309 silesia uncompressed literals old streaming advanced 4842075
1310 silesia uncompressed literals optimal old streaming advanced 4296880 4296686
1311 silesia huffman literals old streaming advanced 6179294
1312 silesia multithreaded with advanced params old streaming advanced 4842075
1313 silesia.tar level -5 old streaming advanced 7043687