From b4250489cf558100c863716a7b3c4cf7ef343a89 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 May 2018 01:53:38 -0700 Subject: [PATCH 1/4] update compression levels for large inputs --- lib/compress/zstd_compress.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 08dc7db2..a979335a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3423,25 +3423,25 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 19, 13, 14, 1, 7, 1, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 1, ZSTD_fast }, /* level 2 */ { 20, 16, 17, 1, 5, 8, ZSTD_dfast }, /* level 3 */ - { 20, 17, 18, 1, 5, 8, ZSTD_dfast }, /* level 4 */ - { 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ - { 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ - { 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */ - { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 20, 18, 18, 1, 5, 8, ZSTD_dfast }, /* level 4 */ + { 20, 18, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 18, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 7 */ + { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 21, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ { 22, 21, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ { 22, 22, 22, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ { 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ - { 23, 22, 22, 4, 4, 48, ZSTD_btopt }, /* level 17 */ - { 23, 22, 22, 5, 3, 64, ZSTD_btopt }, /* level 18 */ - { 23, 23, 22, 7, 3,128, ZSTD_btopt }, /* level 19 */ - { 25, 25, 23, 7, 3,128, ZSTD_btultra }, /* level 20 */ - { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ - { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ + { 23, 22, 22, 4, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3,256, ZSTD_btopt }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra }, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra }, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra }, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra }, /* level 22 */ }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ From c9227ee16bfc95ec99aeb4ca8ddf1217f9129482 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 May 2018 17:15:07 -0700 Subject: [PATCH 2/4] update table for 128 KB blocks --- lib/compress/zstd_compress.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a979335a..622b069b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3471,26 +3471,26 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, T, strat */ - { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* level 0 - not used */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 17, 12, 13, 1, 6, 1, ZSTD_fast }, /* level 1 */ - { 17, 13, 16, 1, 5, 1, ZSTD_fast }, /* level 2 */ - { 17, 16, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ - { 17, 13, 15, 3, 4, 8, ZSTD_greedy }, /* level 4 */ - { 17, 15, 17, 4, 4, 8, ZSTD_greedy }, /* level 5 */ - { 17, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ - { 17, 15, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 13, 15, 1, 5, 1, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 8, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 8, ZSTD_greedy }, /* level 5 */ + { 17, 17, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ + { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ - { 17, 17, 17, 8, 4, 8, ZSTD_lazy2 }, /* level 12 */ - { 17, 18, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13.*/ - { 17, 17, 17, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ - { 17, 17, 17, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ - { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ - { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ - { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ - { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ + { 17, 18, 17, 6, 4, 16, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 8, 4, 16, ZSTD_btlazy2 }, /* level 13.*/ + { 17, 18, 17, 4, 4, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 7, 3,128, ZSTD_btopt }, /* level 16.*/ + { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 17.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 18.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 19.*/ { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ From 63eeeaa1dd60022e657c95317bdd15aeae4436d9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 May 2018 16:13:37 -0700 Subject: [PATCH 3/4] update table levels for blocks <= 16K also : allow hlog to be slighly larger than windowlog, as it's apparently good for both speed and compression ratio. --- lib/compress/zstd_compress.c | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 622b069b..d4e3462d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -743,7 +743,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameter ZSTD_highbit32(tSize-1) + 1; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; } - if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + if (cPar.hashLog > cPar.windowLog+1) cPar.hashLog = cPar.windowLog+1; { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); @@ -3422,11 +3422,11 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 1, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 1, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 5, 8, ZSTD_dfast }, /* level 3 */ - { 20, 18, 18, 1, 5, 8, ZSTD_dfast }, /* level 4 */ - { 20, 18, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ - { 21, 18, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ - { 21, 18, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 7 */ + { 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ + { 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ + { 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 18, 19, 3, 5, 8, ZSTD_lazy2 }, /* level 7 */ { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ { 21, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ @@ -3447,12 +3447,12 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /* W, C, H, S, L, T, strat */ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 18, 13, 14, 1, 6, 1, ZSTD_fast }, /* level 1 */ - { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ - { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ - { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ - { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ - { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ - { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 14, 13, 1, 5, 1, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 1, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */ { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ @@ -3474,10 +3474,10 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 17, 12, 13, 1, 6, 1, ZSTD_fast }, /* level 1 */ { 17, 13, 15, 1, 5, 1, ZSTD_fast }, /* level 2 */ - { 17, 15, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ - { 17, 17, 17, 2, 4, 8, ZSTD_dfast }, /* level 4 */ - { 17, 16, 17, 3, 4, 8, ZSTD_greedy }, /* level 5 */ - { 17, 17, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ + { 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ @@ -3498,25 +3498,25 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 14, 14, 14, 1, 6, 1, ZSTD_fast }, /* level 1 */ - { 14, 14, 14, 1, 4, 1, ZSTD_fast }, /* level 2 */ - { 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ - { 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ - { 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ - { 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ - { 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ - { 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ - { 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ - { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ - { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ + { 14, 14, 15, 1, 5, 1, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 1, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ { 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ { 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ { 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ - { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ - { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 18.*/ + { 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/ { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ { 14, 15, 15, 10, 3,256, ZSTD_btultra }, /* level 22.*/ From b0b3fb517dba3ff3ed4f4d45652f2ff9cc94c3c6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 18 May 2018 17:17:12 -0700 Subject: [PATCH 4/4] updated compression levels for blocks of 256KB --- lib/compress/zstd_compress.c | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d4e3462d..faecad40 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3447,27 +3447,27 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /* W, C, H, S, L, T, strat */ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 18, 13, 14, 1, 6, 1, ZSTD_fast }, /* level 1 */ - { 18, 14, 13, 1, 5, 1, ZSTD_dfast }, /* level 2 */ - { 18, 16, 15, 1, 5, 1, ZSTD_dfast }, /* level 3 */ - { 18, 15, 17, 1, 5, 2, ZSTD_greedy }, /* level 4.*/ - { 18, 16, 17, 4, 5, 2, ZSTD_greedy }, /* level 5.*/ - { 18, 16, 17, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ - { 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ - { 18, 18, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 12.*/ - { 18, 19, 17, 7, 4, 8, ZSTD_btlazy2 }, /* level 13 */ - { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ - { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ - { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ - { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ - { 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/ - { 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/ - { 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/ + { 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 16, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 6, 4, 16, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 19, 19, 8, 4, 16, ZSTD_btlazy2 }, /* level 13 */ + { 18, 18, 19, 4, 4, 24, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 4, 3, 24, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3, 64, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 19, 8, 3,128, ZSTD_btopt }, /* level 17.*/ + { 18, 19, 19, 10, 3,256, ZSTD_btopt }, /* level 18.*/ + { 18, 19, 19, 10, 3,256, ZSTD_btultra }, /* level 19.*/ + { 18, 19, 19, 11, 3,512, ZSTD_btultra }, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra }, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra }, /* level 22.*/ }, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, T, strat */