diff --git a/lib/README.md b/lib/README.md index a0445548..91ef00c4 100644 --- a/lib/README.md +++ b/lib/README.md @@ -13,6 +13,11 @@ The __lib__ directory contains several files, but depending on target use case, ##### zstd core compression +Stable API is exposed in [zstd.h]. +Advanced and experimental API is exposed in `zstd_static.h`. +`zstd_static.h` API elements should be used with static linking only, +as their definition may change in future version of the library. + - [bitstream.h](bitstream.h) - fse.c - fse.h @@ -24,13 +29,15 @@ The __lib__ directory contains several files, but depending on target use case, - zstd_decompress.c - zstd_internal.h - zstd_opt.h -- zstd.h +- [zstd.h] - zstd_static.h +[zstd.h]: zstd.h + #### Buffered streaming This complementary API makes streaming integration easier. -It is used by `zstd` command line utility : +It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) : - zbuff.c - zbuff.h diff --git a/lib/bitstream.h b/lib/bitstream.h index e0487e87..006a927f 100644 --- a/lib/bitstream.h +++ b/lib/bitstream.h @@ -185,7 +185,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t ma MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) { - static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF }; /* up to 25 bits */ + static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos; bitC->bitPos += nbBits; } diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 50979ce5..09303195 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -74,7 +74,6 @@ size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + /*-************************************* * Sequence storage ***************************************/ - static void ZSTD_resetSeqStore(seqStore_t* ssPtr) { ssPtr->offset = ssPtr->offsetStart; @@ -129,7 +128,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) return 0; /* reserved as a potential error code in the future */ } -seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx) +seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ { return ctx->seqStore; } @@ -170,11 +169,26 @@ void ZSTD_validateParams(ZSTD_parameters* params) } +size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for paramagrill */ +{ /* copy / pasted from ZSTD_resetCCtx_advanced */ + const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; + const U32 divider = (params.searchLength==3) ? 3 : 4; + const size_t maxNbSeq = blockSize / divider; + const size_t tokenSpace = blockSize + 8*maxNbSeq; + const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog) + (1 << params.hashLog3)) * sizeof(U32); + const size_t optSpace = ((1<dumps, (U32)((litLength<<1)+1)); seqStorePtr->dumps += 3; - } - } } + } } } else *(seqStorePtr->litLength++) = (BYTE)litLength; /* match offset */ @@ -776,8 +789,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B } else { MEM_writeLE32(seqStorePtr->dumps, (U32)((matchCode<<1)+1)); seqStorePtr->dumps += 3; - } - } } + } } } else *(seqStorePtr->matchLength++) = (BYTE)matchCode; } @@ -1957,7 +1969,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, op += cSize; } - #if ZSTD_OPT_DEBUG == 3 ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4); printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc); @@ -2284,7 +2295,7 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_MAX_CLEVEL 25 +#define ZSTD_MAX_CLEVEL 22 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } @@ -2295,28 +2306,25 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 19, 13, 14, 0, 1, 7, 4, ZSTD_fast }, /* level 1 */ { 0, 19, 15, 16, 0, 1, 6, 4, ZSTD_fast }, /* level 2 */ { 0, 20, 18, 20, 0, 1, 6, 4, ZSTD_fast }, /* level 3 */ - { 0, 21, 19, 21, 0, 1, 6, 4, ZSTD_fast }, /* level 4 */ - { 0, 20, 14, 18, 0, 3, 5, 4, ZSTD_greedy }, /* level 5 */ - { 0, 20, 18, 19, 0, 3, 5, 4, ZSTD_greedy }, /* level 6 */ + { 0, 20, 13, 17, 0, 2, 5, 4, ZSTD_greedy }, /* level 4.*/ + { 0, 20, 15, 18, 0, 3, 5, 4, ZSTD_greedy }, /* level 5 */ + { 0, 21, 16, 19, 0, 2, 5, 4, ZSTD_lazy }, /* level 6 */ { 0, 21, 17, 20, 0, 3, 5, 4, ZSTD_lazy }, /* level 7 */ - { 0, 21, 19, 20, 0, 3, 5, 4, ZSTD_lazy }, /* level 8 */ + { 0, 21, 18, 20, 0, 3, 5, 4, ZSTD_lazy2 }, /* level 8.*/ { 0, 21, 20, 20, 0, 3, 5, 4, ZSTD_lazy2 }, /* level 9 */ { 0, 21, 19, 21, 0, 4, 5, 4, ZSTD_lazy2 }, /* level 10 */ { 0, 22, 20, 22, 0, 4, 5, 4, ZSTD_lazy2 }, /* level 11 */ { 0, 22, 20, 22, 0, 5, 5, 4, ZSTD_lazy2 }, /* level 12 */ { 0, 22, 21, 22, 0, 5, 5, 4, ZSTD_lazy2 }, /* level 13 */ - { 0, 22, 22, 23, 0, 5, 5, 4, ZSTD_lazy2 }, /* level 14 */ - { 0, 23, 23, 23, 0, 5, 5, 4, ZSTD_lazy2 }, /* level 15 */ + { 0, 22, 21, 22, 0, 6, 5, 4, ZSTD_lazy2 }, /* level 14 */ + { 0, 22, 21, 21, 0, 5, 5, 4, ZSTD_btlazy2 }, /* level 15 */ { 0, 23, 22, 22, 0, 5, 5, 4, ZSTD_btlazy2 }, /* level 16 */ - { 0, 24, 24, 23, 0, 4, 5, 4, ZSTD_btlazy2 }, /* level 17 */ - { 0, 24, 24, 23, 0, 5, 5, 30, ZSTD_btopt }, /* level 18 */ - { 0, 25, 25, 24, 0, 5, 4, 40, ZSTD_btopt }, /* level 19 */ - { 0, 26, 26, 25, 0, 8, 4,256, ZSTD_btopt }, /* level 20 */ - { 0, 26, 27, 25, 0, 10, 4,256, ZSTD_btopt }, /* level 21 */ - { 0, 24, 24, 23, 16, 5, 3, 30, ZSTD_btopt }, /* level 22 */ - { 0, 25, 25, 24, 16, 5, 3, 40, ZSTD_btopt }, /* level 23 */ - { 0, 26, 26, 25, 16, 8, 3,256, ZSTD_btopt }, /* level 24 */ - { 0, 26, 27, 25, 24, 10, 3,256, ZSTD_btopt }, /* level 25 */ + { 0, 23, 22, 22, 0, 6, 5, 22, ZSTD_btopt }, /* level 17 */ + { 0, 22, 22, 22, 0, 5, 3, 44, ZSTD_btopt }, /* level 18 */ + { 0, 23, 24, 22, 0, 7, 3, 44, ZSTD_btopt }, /* level 19 */ + { 0, 25, 26, 22, 0, 7, 3, 71, ZSTD_btopt }, /* level 20 */ + { 0, 26, 26, 24, 0, 7, 3,256, ZSTD_btopt }, /* level 21 */ + { 0, 27, 28, 26, 0, 9, 3,256, ZSTD_btopt }, /* level 22 */ }, { /* for srcSize <= 256 KB */ /* l, W, C, H, H3, S, L, T, strat */ @@ -2343,9 +2351,6 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 18, 19, 18, 0, 11, 4,256, ZSTD_btopt }, /* level 20.*/ { 0, 18, 19, 18, 0, 12, 4,256, ZSTD_btopt }, /* level 21.*/ { 0, 18, 19, 18, 0, 12, 4,256, ZSTD_btopt }, /* level 21-2*/ - { 0, 18, 19, 18, 0, 12, 4,256, ZSTD_btopt }, /* level 21-3*/ - { 0, 18, 19, 18, 0, 12, 4,256, ZSTD_btopt }, /* level 21-4*/ - { 0, 18, 19, 18, 0, 12, 4,256, ZSTD_btopt }, /* level 21-5*/ }, { /* for srcSize <= 128 KB */ /* l, W, C, H, H3, S, L, T, strat */ @@ -2356,26 +2361,22 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 17, 13, 15, 0, 3, 4, 4, ZSTD_greedy }, /* level 4 */ { 0, 17, 15, 17, 0, 4, 4, 4, ZSTD_greedy }, /* level 5 */ { 0, 17, 16, 17, 0, 3, 4, 4, ZSTD_lazy }, /* level 6 */ - { 0, 17, 16, 17, 0, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 0, 17, 17, 16, 0, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */ - { 0, 17, 17, 16, 0, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */ - { 0, 17, 17, 16, 0, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */ + { 0, 17, 15, 17, 0, 4, 4, 4, ZSTD_lazy2 }, /* level 7 */ + { 0, 17, 17, 17, 0, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */ + { 0, 17, 17, 17, 0, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */ + { 0, 17, 17, 17, 0, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */ { 0, 17, 17, 17, 0, 7, 4, 4, ZSTD_lazy2 }, /* level 11 */ { 0, 17, 17, 17, 0, 8, 4, 4, ZSTD_lazy2 }, /* level 12 */ - { 0, 17, 17, 17, 0, 9, 4, 4, ZSTD_lazy2 }, /* level 13 */ - { 0, 17, 18, 16, 0, 5, 4, 20, ZSTD_btopt }, /* level 14 */ - { 0, 17, 18, 16, 0, 9, 4, 48, ZSTD_btopt }, /* level 15 */ - { 0, 17, 18, 17, 0, 7, 4,128, ZSTD_btopt }, /* level 16 */ - { 0, 17, 18, 17, 0, 8, 4,128, ZSTD_btopt }, /* level 17 */ - { 0, 17, 18, 17, 0, 8, 4,256, ZSTD_btopt }, /* level 18 */ - { 0, 17, 18, 17, 16, 9, 3,256, ZSTD_btopt }, /* level 19 */ - { 0, 17, 18, 17, 0, 10, 4,512, ZSTD_btopt }, /* level 20 */ - { 0, 17, 18, 17, 0, 11, 4,512, ZSTD_btopt }, /* level 21 */ - { 0, 17, 18, 17, 0, 11, 4,512, ZSTD_btopt }, /* level 21-2 */ - { 0, 17, 18, 17, 0, 11, 4,512, ZSTD_btopt }, /* level 21-3 */ - { 0, 17, 18, 17, 0, 11, 4,512, ZSTD_btopt }, /* level 21-4 */ - { 0, 17, 18, 17, 0, 11, 4,512, ZSTD_btopt }, /* level 21-5 */ - + { 0, 17, 18, 17, 0, 6, 4, 4, ZSTD_btlazy2 }, /* level 13.*/ + { 0, 17, 17, 17, 16, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ + { 0, 17, 17, 17, 16, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 0, 17, 18, 17, 16, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 0, 17, 18, 17, 16, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 0, 17, 18, 17, 16, 7, 3,256, ZSTD_btopt }, /* level 18.*/ + { 0, 17, 18, 17, 16, 8, 3,256, ZSTD_btopt }, /* level 19.*/ + { 0, 17, 18, 17, 16, 9, 3,256, ZSTD_btopt }, /* level 20.*/ + { 0, 17, 18, 17, 16, 10, 3,256, ZSTD_btopt }, /* level 21.*/ + { 0, 17, 18, 17, 16, 11, 3,256, ZSTD_btopt }, /* level 22.*/ }, { /* for srcSize <= 16 KB */ /* l, W, C, H, H3, S, L, T, strat */ @@ -2402,9 +2403,6 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 14, 15, 15, 16, 8, 3,256, ZSTD_btopt }, /* level 20.*/ { 0, 14, 15, 15, 16, 9, 3,256, ZSTD_btopt }, /* level 21.*/ { 0, 14, 15, 15, 16, 10, 3,256, ZSTD_btopt }, /* level 22.*/ - { 0, 14, 15, 15, 16, 11, 3,256, ZSTD_btopt }, /* level 23.*/ - { 0, 14, 15, 15, 16, 12, 3,256, ZSTD_btopt }, /* level 24.*/ - { 0, 14, 15, 15, 16, 13, 3,256, ZSTD_btopt }, /* level 25.*/ }, }; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 844aa976..60136b61 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -648,9 +648,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) /* Offset */ { static const U32 offsetPrefix[MaxOff+1] = { - 1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256, - 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, - 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 }; + 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100, + 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, + 0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 }; U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ U32 nbBits = offsetCode - 1; if (offsetCode==0) nbBits = 0; /* cmove */ diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 130eb210..ec7e72f2 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -91,6 +91,7 @@ static const size_t ZSTD_frameHeaderSize_min = 5; #define MINMATCH 4 #define REPCODE_STARTVALUE 1 +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12 #define Litbits 8 #define MLbits 7 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 0effaa7c..40522598 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -57,20 +57,19 @@ extern "C" { /*-************************************* * Types ***************************************/ -#define ZSTD_WINDOWLOG_MAX 26 -#define ZSTD_WINDOWLOG_MIN 18 -#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11 -#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CONTENTLOG_MIN 4 -#define ZSTD_HASHLOG_MAX 28 -#define ZSTD_HASHLOG_MIN 12 -#define ZSTD_HASHLOG3_MAX 24 -#define ZSTD_HASHLOG3_MIN 2 -#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_SEARCHLENGTH_MAX 7 -#define ZSTD_SEARCHLENGTH_MIN 3 -#define ZSTD_TARGETLENGTH_MIN 4 +#define ZSTD_WINDOWLOG_MAX 27 +#define ZSTD_WINDOWLOG_MIN 18 +#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CONTENTLOG_MIN 4 +#define ZSTD_HASHLOG_MAX 28 +#define ZSTD_HASHLOG_MIN 12 +#define ZSTD_HASHLOG3_MAX 16 +#define ZSTD_HASHLOG3_MIN 16 +#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_SEARCHLENGTH_MAX 7 +#define ZSTD_SEARCHLENGTH_MIN 3 +#define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MAX 999 /* from faster to stronger */ diff --git a/programs/.gitignore b/programs/.gitignore index 2fc85021..525037b9 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -6,6 +6,7 @@ fullbench32 fuzzer fuzzer32 datagen +paramgrill # Object files *.o @@ -30,5 +31,6 @@ datagen *.suo *.user -# Default dictionary name +# Default result files dictionary +grillResults.txt diff --git a/programs/paramgrill.c b/programs/paramgrill.c index db8ff0e7..4a76da8a 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -447,6 +447,7 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz BMK_printWinners2(stdout, winners, srcSize); } +size_t ZSTD_sizeofCCtx(ZSTD_parameters params); /* hidden interface, declared here */ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, const void* srcBuffer, size_t srcSize, @@ -481,10 +482,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) + - ((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog)); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) + - ((winners[cLevel].params.strategy==ZSTD_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog)); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_sizeofCCtx(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_sizeofCCtx(winners[cLevel].params); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);