facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'advancedAPI2' into refPrefix

dev
Yann Collet 2017-06-28 16:35:49 -07:00
parent 62f7efc04a 1ca76039af
commit 2bf428df45
3 changed files with 68 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/zstd_manual.html
View File

@ -412,9 +412,9 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDCtxSize(void);
</b><p> These functions make it possible to estimate memory usage
of a future {D,C}Ctx, before its creation.
The objective is to guide decision before allocation.
ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
Note : CCtx estimation is only correct for single-threaded compression
</p></pre><BR>
@ -423,8 +423,9 @@ size_t ZSTD_estimateDCtxSize(void);
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDStreamSize(size_t windowSize);
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
</b><p> ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
</b><p> ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
Note : CStream estimation is only correct for single-threaded compression.
ZSTD_DStream memory budget depends on window Size.

66
lib/compress/zstd_compress.c
View File

@ -4009,30 +4009,30 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" */
{ /* "default" - guarantees a monotonically increasing memory budget */
/* W, C, H, S, L, TL, strat */
{ 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */
{ 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */
{ 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/
{ 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/
{ 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */
{ 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
{ 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */
{ 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */
{ 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */
{ 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */
{ 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
{ 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */
{ 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */
{ 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
{ 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */
{ 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */
{ 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */
{ 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */
{ 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */
{ 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */
{ 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
{ 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */
{ 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
{ 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */
{ 26, 26, 23, 7, 3,256, ZSTD_btultra }, /* level 21 */
{ 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */
{ 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */
},
{ /* for srcSize <= 256 KB */
@ -4115,6 +4115,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
/* This function just controls
* the monotonic memory budget increase of ZSTD_defaultCParameters[0].
* Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
*/
MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
{
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
}
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
assert(ZSTD_maxCLevel()==22);
}
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
* Size values are optional, provide 0 if not known or unused */
@ -4123,6 +4160,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
size_t const addedSize = srcSizeHint ? 0 : 500;
U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
static int g_monotonicTest = 1;
if (g_monotonicTest) {
ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget();
g_monotonicTest=0;
}
#endif
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
{ ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];

11
lib/zstd.h
View File

@ -497,9 +497,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
* of a future {D,C}Ctx, before its creation.
* The objective is to guide decision before allocation.
* ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
* If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
@ -507,8 +507,9 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cPa
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
/*! ZSTD_estimate?StreamSize() :
* ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
* If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size.