From 2f10019b92128bbc6098ab77ea3865ff496221ad Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 14:12:03 -0800 Subject: [PATCH 01/14] Adding --show-default-cparams (show cparams before compressing --- lib/compress/zstd_compress.c | 108 ------------------------ lib/compress/zstd_compress_internal.h | 113 ++++++++++++++++++++++++++ programs/zstdcli.c | 33 ++++++++ tests/playTests.sh | 12 +++ 4 files changed, 158 insertions(+), 108 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e11092b4..562893d2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -4071,117 +4071,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_MAX_CLEVEL 22 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } -static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" - for any srcSize > 256 KB */ - /* W, C, H, S, L, TL, strat */ - { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ - { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ - { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ - { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ - { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ - { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ - { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ - { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ - { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ - { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ - { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ - { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ - { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ - { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ - { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ - { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ - { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ - { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ - { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ -}, -{ /* for srcSize <= 256 KB */ - /* W, C, H, S, L, T, strat */ - { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ - { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ - { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ - { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ - { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ - { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ - { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ - { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ - { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ - { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ - { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ - { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -{ /* for srcSize <= 128 KB */ - /* W, C, H, S, L, T, strat */ - { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ - { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ - { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ - { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ - { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ - { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ - { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ - { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ - { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ - { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ - { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ - { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ - { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ - { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -{ /* for srcSize <= 16 KB */ - /* W, C, H, S, L, T, strat */ - { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ - { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ - { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ - { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ - { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ - { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ - { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ - { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ - { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ - { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ - { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ - { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ - { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ - { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ - { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ - { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ - { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ - { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -}; - /*! ZSTD_getCParams_internal() : * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 243f0e07..ae984b4b 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -551,6 +551,119 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); } +/*-************************************* + * Compression parameters + ***************************************/ + +#define ZSTD_MAX_CLEVEL 22 + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ + { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + /*-************************************* * Hashes diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f57bf3c0..35191ad7 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -41,6 +41,7 @@ # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ #endif #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */ +#include "zstd_compress_internal.h" /* ZSTD_compressionParameters, ZSTD_strategyMap */ /*-************************************ @@ -580,6 +581,7 @@ int main(int const argCount, const char* argv[]) separateFiles = 0, setRealTimePrio = 0, singleThread = 0, + showDefaultCParams = 0, ultra=0; double compressibility = 0.5; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ @@ -693,6 +695,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } + if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } @@ -1170,6 +1173,17 @@ int main(int const argCount, const char* argv[]) } } #endif + if (showDefaultCParams) { + if (operation == zom_decompress) { + DISPLAY("error : can't use --show-default-cparams in decomrpession mode \n"); + CLEAN_RETURN(1); + } + if (!UTIL_isRegularFile(filenames->fileNames[0])) { + DISPLAY("error : can't use --show-default-cparams with non-regular files \n"); + CLEAN_RETURN(1); + } + } + if (dictFileName != NULL && patchFromDictFileName != NULL) { DISPLAY("error : can't use -D and --patch-from=# at the same time \n"); CLEAN_RETURN(1); @@ -1213,6 +1227,25 @@ int main(int const argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + if (showDefaultCParams) { + size_t fileNb; + static const char* ZSTD_strategyMap[9 + 1] = { "", "ZSTD_fast", + "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", + "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; + for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { + const size_t fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]); + const size_t cParamsIdx = (fileSize < 16 KB) ? 3 : ( (fileSize < 128 KB) ? 2 : ( (fileSize < 256 KB) ? 1 : 0) ); + ZSTD_compressionParameters cParams = ZSTD_defaultCParameters[cParamsIdx][cLevel]; + DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); + DISPLAY(" - windowLog : %u\n", (unsigned)cParams.windowLog); + DISPLAY(" - chainLog : %u\n", (unsigned)cParams.chainLog); + DISPLAY(" - searchLog : %u\n", (unsigned)cParams.searchLog); + DISPLAY(" - minMatch : %u\n", (unsigned)cParams.minMatch); + DISPLAY(" - targetLength : %u\n", (unsigned)cParams.targetLength); + DISPLAY(" - strategy : %s\n", ZSTD_strategyMap[(int)cParams.strategy]); + } + } + if ((filenames->tableSize==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else diff --git a/tests/playTests.sh b/tests/playTests.sh index 6ae5bd8c..09fe92fc 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -457,6 +457,18 @@ ls tmp* > tmpList $ZSTD -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list rm -rf tmp* +println "test : show-default-cparams regular" +$DATAGEN > tmp +$ZSTD --show-default-cparams -f tmp +rm -rf tmp* + +println "test : show-default-cparams recursive" +mkdir tmp_files +$DATAGEN -g15000 > tmp_files/tmp1 +$DATAGEN -g129000 > tmp_files/tmp2 +$DATAGEN -g257000 > tmp_files/tmp3 +$ZSTD --show-default-cparams -f -r tmp_files +rm -rf tmp* println "\n===> Advanced compression parameters " println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" From 71754a26f85c2885fbdb116668307285333651d6 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 14:14:27 -0800 Subject: [PATCH 02/14] Update comment --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 35191ad7..00ec7996 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -41,7 +41,7 @@ # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ #endif #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */ -#include "zstd_compress_internal.h" /* ZSTD_compressionParameters, ZSTD_strategyMap */ +#include "zstd_compress_internal.h" /* ZSTD_compressionParameters */ /*-************************************ From b660ef87f85646891211ce162c0dd1b38eba366e Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 14:29:04 -0800 Subject: [PATCH 03/14] Convertion nit U32 -> size_t --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 00ec7996..b2a47e78 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1233,7 +1233,7 @@ int main(int const argCount, const char* argv[]) "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { - const size_t fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]); + const size_t fileSize = (size_t)UTIL_getFileSize(filenames->fileNames[fileNb]); const size_t cParamsIdx = (fileSize < 16 KB) ? 3 : ( (fileSize < 128 KB) ? 2 : ( (fileSize < 256 KB) ? 1 : 0) ); ZSTD_compressionParameters cParams = ZSTD_defaultCParameters[cParamsIdx][cLevel]; DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); From b0ceab4dc061ea82c38b899ab5dfe6a8b577841b Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 16:52:07 -0800 Subject: [PATCH 04/14] Moving default cparams back, checking for unknown file size, using getCParams() --- lib/compress/zstd_compress.c | 108 ++++++++++++++++++++++++ lib/compress/zstd_compress_internal.h | 113 -------------------------- programs/zstdcli.c | 22 +++-- 3 files changed, 118 insertions(+), 125 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 562893d2..e11092b4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -4071,9 +4071,117 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /*-===== Pre-defined compression levels =====-*/ +#define ZSTD_MAX_CLEVEL 22 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ + { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + /*! ZSTD_getCParams_internal() : * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index ae984b4b..243f0e07 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -551,119 +551,6 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd); } -/*-************************************* - * Compression parameters - ***************************************/ - -#define ZSTD_MAX_CLEVEL 22 - -static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" - for any srcSize > 256 KB */ - /* W, C, H, S, L, TL, strat */ - { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ - { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ - { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ - { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ - { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ - { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ - { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ - { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ - { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ - { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ - { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ - { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ - { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ - { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ - { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ - { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ - { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ - { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ - { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ -}, -{ /* for srcSize <= 256 KB */ - /* W, C, H, S, L, T, strat */ - { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ - { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ - { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ - { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ - { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ - { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ - { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ - { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ - { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ - { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ - { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ - { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -{ /* for srcSize <= 128 KB */ - /* W, C, H, S, L, T, strat */ - { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ - { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ - { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ - { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ - { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ - { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ - { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ - { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ - { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ - { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ - { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ - { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ - { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ - { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ - { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ - { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -{ /* for srcSize <= 16 KB */ - /* W, C, H, S, L, T, strat */ - { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ - { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ - { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ - { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ - { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ - { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ - { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ - { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ - { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ - { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ - { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ - { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ - { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ - { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ - { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ - { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ - { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ - { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ - { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ - { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ - { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ - { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ - { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ -}, -}; - /*-************************************* * Hashes diff --git a/programs/zstdcli.c b/programs/zstdcli.c index b2a47e78..8936f2c5 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -41,7 +41,6 @@ # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ #endif #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */ -#include "zstd_compress_internal.h" /* ZSTD_compressionParameters */ /*-************************************ @@ -545,6 +544,11 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } +#define ZSTD_NB_STRATEGIES 9 +static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", + "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", + "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; + typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -1178,10 +1182,6 @@ int main(int const argCount, const char* argv[]) DISPLAY("error : can't use --show-default-cparams in decomrpession mode \n"); CLEAN_RETURN(1); } - if (!UTIL_isRegularFile(filenames->fileNames[0])) { - DISPLAY("error : can't use --show-default-cparams with non-regular files \n"); - CLEAN_RETURN(1); - } } if (dictFileName != NULL && patchFromDictFileName != NULL) { @@ -1229,14 +1229,12 @@ int main(int const argCount, const char* argv[]) if (showDefaultCParams) { size_t fileNb; - static const char* ZSTD_strategyMap[9 + 1] = { "", "ZSTD_fast", - "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", - "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { - const size_t fileSize = (size_t)UTIL_getFileSize(filenames->fileNames[fileNb]); - const size_t cParamsIdx = (fileSize < 16 KB) ? 3 : ( (fileSize < 128 KB) ? 2 : ( (fileSize < 256 KB) ? 1 : 0) ); - ZSTD_compressionParameters cParams = ZSTD_defaultCParameters[cParamsIdx][cLevel]; - DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); + const size_t fileSize = UTIL_isRegularFile(filenames->fileNames[fileNb]) ? (size_t)UTIL_getFileSize(filenames->fileNames[fileNb]) : 0; + const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; + const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, (unsigned long long)fileSize, dictSize); + if (fileSize) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); + else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]); DISPLAY(" - windowLog : %u\n", (unsigned)cParams.windowLog); DISPLAY(" - chainLog : %u\n", (unsigned)cParams.chainLog); DISPLAY(" - searchLog : %u\n", (unsigned)cParams.searchLog); From 2a0771cfdbe5eccc9e097b2368906ec8d99a4674 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:05:33 -0800 Subject: [PATCH 05/14] Removing unnecessary cast --- programs/zstdcli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 8936f2c5..3921efba 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1230,9 +1230,9 @@ int main(int const argCount, const char* argv[]) if (showDefaultCParams) { size_t fileNb; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { - const size_t fileSize = UTIL_isRegularFile(filenames->fileNames[fileNb]) ? (size_t)UTIL_getFileSize(filenames->fileNames[fileNb]) : 0; + unsigned long long fileSize = UTIL_isRegularFile(filenames->fileNames[fileNb]) ? UTIL_getFileSize(filenames->fileNames[fileNb]) : 0; const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; - const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, (unsigned long long)fileSize, dictSize); + const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); if (fileSize) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]); DISPLAY(" - windowLog : %u\n", (unsigned)cParams.windowLog); From c2033412928384c113411aa7430e6bb2956bdb34 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:07:22 -0800 Subject: [PATCH 06/14] Removing more casts --- programs/zstdcli.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3921efba..aa6df7b6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1233,13 +1233,13 @@ int main(int const argCount, const char* argv[]) unsigned long long fileSize = UTIL_isRegularFile(filenames->fileNames[fileNb]) ? UTIL_getFileSize(filenames->fileNames[fileNb]) : 0; const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); - if (fileSize) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); + if (fileSize) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], fileSize); else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]); - DISPLAY(" - windowLog : %u\n", (unsigned)cParams.windowLog); - DISPLAY(" - chainLog : %u\n", (unsigned)cParams.chainLog); - DISPLAY(" - searchLog : %u\n", (unsigned)cParams.searchLog); - DISPLAY(" - minMatch : %u\n", (unsigned)cParams.minMatch); - DISPLAY(" - targetLength : %u\n", (unsigned)cParams.targetLength); + DISPLAY(" - windowLog : %u\n", cParams.windowLog); + DISPLAY(" - chainLog : %u\n", cParams.chainLog); + DISPLAY(" - searchLog : %u\n", cParams.searchLog); + DISPLAY(" - minMatch : %u\n", cParams.minMatch); + DISPLAY(" - targetLength : %u\n", cParams.targetLength); DISPLAY(" - strategy : %s\n", ZSTD_strategyMap[(int)cParams.strategy]); } } From 6bf05a53ab9db1aeb6a316161f479d68f148b90a Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:11:38 -0800 Subject: [PATCH 07/14] Documenting in the man file --- programs/zstd.1.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 8539deb9..5997ed6a 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -363,6 +363,11 @@ Compression of small files similar to the sample set will be greatly improved. `zstd --train-legacy=selectivity=8 FILEs` +* `--show-default-cparams`: + Shows the default compresssion parameters that will be used for a + particular src file. If the provided src file is not a regular file + (eg. named pipe), the cli will just output the default paramters. + BENCHMARK --------- From 6ad2f0d7d670afb644ba27a7f38f51bb9bb9d501 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:16:02 -0800 Subject: [PATCH 08/14] Adding back one cast and using UTIL_FILESIZE_UNKNOWN to check unknown file size --- programs/zstdcli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index aa6df7b6..40d1e621 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1230,10 +1230,10 @@ int main(int const argCount, const char* argv[]) if (showDefaultCParams) { size_t fileNb; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { - unsigned long long fileSize = UTIL_isRegularFile(filenames->fileNames[fileNb]) ? UTIL_getFileSize(filenames->fileNames[fileNb]) : 0; + unsigned long long fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]); const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); - if (fileSize) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], fileSize); + if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]); DISPLAY(" - windowLog : %u\n", cParams.windowLog); DISPLAY(" - chainLog : %u\n", cParams.chainLog); From 9d9cfa9ef9d92374c6f0fb59c16c3634fa0511f5 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:54:01 -0800 Subject: [PATCH 09/14] Adding assert and using ZSTD_STRATEGY_MAX as ZSTD_NB_STRATEGIES --- programs/zstdcli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 40d1e621..fed0a179 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -544,7 +544,7 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } -#define ZSTD_NB_STRATEGIES 9 +#define ZSTD_NB_STRATEGIES ZSTD_STRATEGY_MAX static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; @@ -1240,6 +1240,7 @@ int main(int const argCount, const char* argv[]) DISPLAY(" - searchLog : %u\n", cParams.searchLog); DISPLAY(" - minMatch : %u\n", cParams.minMatch); DISPLAY(" - targetLength : %u\n", cParams.targetLength); + assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1); DISPLAY(" - strategy : %s\n", ZSTD_strategyMap[(int)cParams.strategy]); } } From f4ae6c0b942b9edf5322ff1c008d9f9495ce1f22 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 30 Jan 2020 17:54:42 -0800 Subject: [PATCH 10/14] Moving documentation and elaborating --- programs/zstd.1.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 5997ed6a..7dcc9033 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -248,6 +248,14 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a `ZSTD_CLEVEL` just replaces the default compression level (`3`). It can be overridden by corresponding command line arguments. +### Misc. arguments + +* `--show-default-cparams`: + Shows the default compresssion parameters that will be used for a + particular src file. If the provided src file is not a regular file + (eg. named pipe), the cli will just output the default paramters. + That is, the parameters that are used when the src size is + unknown. DICTIONARY BUILDER ------------------ @@ -363,11 +371,6 @@ Compression of small files similar to the sample set will be greatly improved. `zstd --train-legacy=selectivity=8 FILEs` -* `--show-default-cparams`: - Shows the default compresssion parameters that will be used for a - particular src file. If the provided src file is not a regular file - (eg. named pipe), the cli will just output the default paramters. - BENCHMARK --------- From 0fa6c90e89a4ebbd944e3a75853bfcecd1215dd4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 31 Jan 2020 10:38:19 -0800 Subject: [PATCH 11/14] Moving to same category as -v --- programs/zstd.1.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 7dcc9033..5b6505ba 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -228,6 +228,12 @@ the last one takes effect. `-vvV` also displays POSIX support. * `-v`: verbose mode +* `--show-default-cparams`: + Shows the default compresssion parameters that will be used for a + particular src file. If the provided src file is not a regular file + (eg. named pipe), the cli will just output the default paramters. + That is, the parameters that are used when the src size is + unknown. * `-q`, `--quiet`: suppress warnings, interactivity, and notifications. specify twice to suppress errors too. @@ -248,14 +254,6 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a `ZSTD_CLEVEL` just replaces the default compression level (`3`). It can be overridden by corresponding command line arguments. -### Misc. arguments - -* `--show-default-cparams`: - Shows the default compresssion parameters that will be used for a - particular src file. If the provided src file is not a regular file - (eg. named pipe), the cli will just output the default paramters. - That is, the parameters that are used when the src size is - unknown. DICTIONARY BUILDER ------------------ From 38d010eea048aed776fcea07e958883b18cea174 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 31 Jan 2020 10:47:17 -0800 Subject: [PATCH 12/14] Adding static assert and using it in cli --- programs/zstdcli.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fed0a179..01be5fdc 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -544,7 +544,10 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } +#define ZSTD_CLI_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + #define ZSTD_NB_STRATEGIES ZSTD_STRATEGY_MAX + static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; @@ -1227,6 +1230,8 @@ int main(int const argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + ZSTD_CLI_STATIC_ASSERT(ZSTD_NB_STRATEGIES >= ZSTD_STRATEGY_MIN && ZSTD_NB_STRATEGIES <= ZSTD_STRATEGY_MAX); + if (showDefaultCParams) { size_t fileNb; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { From 19473390d120792f1e2be239fa6e1d33d1e05a15 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 31 Jan 2020 10:54:02 -0800 Subject: [PATCH 13/14] Adding (void)ZSTD_strategyMap --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 01be5fdc..c9187475 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1255,7 +1255,7 @@ int main(int const argCount, const char* argv[]) else operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ + (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ From d1389ac3ffadfc0fd105b62b4c462dba49217727 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 3 Feb 2020 09:52:39 -0800 Subject: [PATCH 14/14] Adding assert(NB_STRATEGIES == upperLimit) --- programs/zstdcli.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c9187475..341f38cb 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -544,9 +544,7 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } -#define ZSTD_CLI_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) - -#define ZSTD_NB_STRATEGIES ZSTD_STRATEGY_MAX +#define ZSTD_NB_STRATEGIES 9 static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", @@ -1230,7 +1228,9 @@ int main(int const argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - ZSTD_CLI_STATIC_ASSERT(ZSTD_NB_STRATEGIES >= ZSTD_STRATEGY_MIN && ZSTD_NB_STRATEGIES <= ZSTD_STRATEGY_MAX); + /* Compare strategies constant with the ground truth */ + { ZSTD_bounds strategyBounds = ZSTD_cParam_getBounds(ZSTD_c_strategy); + assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound);} if (showDefaultCParams) { size_t fileNb;