facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into Other

Browse Source
This commit is contained in:
inikep 2016-08-24 21:25:49 +02:00
commit a3a47ec4d0
6 changed files with 107 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/common/error_private.h
View File

@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
case PREFIX(no_error): return "No error detected"; case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor"; case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";

4
lib/common/error_public.h
View File

@ -37,6 +37,9 @@
extern "C" { extern "C" {
#endif #endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
/* **************************************** /* ****************************************
* error codes list * error codes list
@ -45,6 +48,7 @@ typedef enum {
ZSTD_error_no_error, ZSTD_error_no_error,
ZSTD_error_GENERIC, ZSTD_error_GENERIC,
ZSTD_error_prefix_unknown, ZSTD_error_prefix_unknown,
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits, ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_compressionParameter_unsupported, ZSTD_error_compressionParameter_unsupported,

79
lib/compress/zstd_compress.c
View File

@ -1230,9 +1230,20 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
offset = (U32)(ip-matchLong); offset = (U32)(ip-matchLong);
while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
} else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) {
mLength = ZSTD_count(ip+4, match+4, iend) + 4; size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
offset = (U32)(ip-match); U32 const matchIndex3 = hashLong[h3];
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ const BYTE* match3 = base + matchIndex3;
hashLong[h3] = current + 1;
if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
mLength = ZSTD_count(ip+9, match3+8, iend) + 8;
ip++;
offset = (U32)(ip-match3);
while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
} else {
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
offset = (U32)(ip-match);
while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
}
} else { } else {
ip += ((ip-anchor) >> g_searchStrength) + 1; ip += ((ip-anchor) >> g_searchStrength) + 1;
continue; continue;
@ -1359,16 +1370,32 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) {
const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; U32 const matchIndex3 = hashLong[h3];
const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base;
const BYTE* match3 = match3Base + matchIndex3;
U32 offset; U32 offset;
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; hashLong[h3] = current + 1;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
offset = current - matchIndex; const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8;
ip++;
offset = current+1 - matchIndex3;
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
} else {
const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4;
offset = current - matchIndex;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
}
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else { } else {
ip += ((ip-anchor) >> g_searchStrength) + 1; ip += ((ip-anchor) >> g_searchStrength) + 1;
continue; continue;
@ -3031,7 +3058,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */ { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */
{ 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */
{ 20, 16, 18, 1, 5, 16, ZSTD_dfast }, /* level 3 */ { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */
{ 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/ { 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/
{ 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */
{ 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */
@ -3054,24 +3081,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
}, },
{ /* for srcSize <= 256 KB */ { /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
{ 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */
{ 18, 13, 14, 1, 6, 4, ZSTD_fast }, /* level 1 */ { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */
{ 18, 15, 17, 1, 5, 4, ZSTD_fast }, /* level 2 */ { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */
{ 18, 13, 15, 1, 5, 4, ZSTD_greedy }, /* level 3.*/ { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */
{ 18, 15, 17, 1, 5, 4, ZSTD_greedy }, /* level 4.*/ { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/
{ 18, 16, 17, 4, 5, 4, ZSTD_greedy }, /* level 5 */ { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/
{ 18, 17, 17, 5, 5, 4, ZSTD_greedy }, /* level 6 */ { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/
{ 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */ { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */
{ 18, 17, 17, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */ { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 18, 17, 17, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */ { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 18, 17, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */ { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 18, 18, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 11.*/ { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/
{ 18, 18, 17, 7, 4, 4, ZSTD_lazy2 }, /* level 12.*/ { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/
{ 18, 19, 17, 7, 4, 4, ZSTD_btlazy2 }, /* level 13 */ { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */
{ 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/
{ 18, 18, 18, 8, 4, 24, ZSTD_btopt }, /* level 15.*/ { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 18, 8, 3, 48, ZSTD_btopt }, /* level 16.*/ { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/
{ 18, 19, 18, 8, 3, 96, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
{ 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/ { 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/

26
lib/decompress/zstd_decompress.c
View File

@ -51,12 +51,12 @@
/*! /*!
* STREAM_WINDOW_MAX : * MAXWINDOWSIZE_DEFAULT :
* maximum window size accepted by DStream. * maximum window size accepted by DStream, by default.
* frames requiring more memory will be rejected. * Frames requiring more memory will be rejected.
*/ */
#ifndef ZSTD_STREAM_WINDOW_MAX #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ # define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */
#endif #endif
@ -1328,6 +1328,7 @@ struct ZSTD_DStream_s {
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inPos; size_t inPos;
size_t maxWindowSize;
char* outBuff; char* outBuff;
size_t outBuffSize; size_t outBuffSize;
size_t outStart; size_t outStart;
@ -1361,6 +1362,7 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
zds->zd = ZSTD_createDCtx_advanced(customMem); zds->zd = ZSTD_createDCtx_advanced(customMem);
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init; zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds; return zds;
} }
@ -1392,6 +1394,18 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0); return ZSTD_initDStream_usingDict(zds, NULL, 0);
} }
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
ZSTD_DStreamParameter_e paramType, unsigned paramValue)
{
switch(paramType)
{
default : return ERROR(parameter_unknown);
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
}
return 0;
}
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{ {
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
@ -1450,11 +1464,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} } } }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
/* Frame header instruct buffer sizes */ /* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize; size_t const neededOutSize = zds->fParams.windowSize + blockSize;
if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported);
zds->blockSize = blockSize; zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) { if (zds->inBuffSize < blockSize) {
zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);

10
lib/zstd.h
View File

@ -87,7 +87,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
* When `return==0`, data to decompress can have any size. * When `return==0`, data to decompress can have any size.
* In which case, it's necessary to use streaming mode to decompress data. * In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application may rely on its own implied limits. * Optionally, application may rely on its own implied limits.
* (For example, application own data could be necessarily cut into blocks <= 16 KB). * (For example, application data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified ! * note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits ! * Always ensure result fits within application's authorized limits !
* Each application can set its own limits. * Each application can set its own limits.
@ -264,7 +264,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* *
* A ZSTD_DStream object is required to track streaming operations. * A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be re-init multiple times. * ZSTD_DStream objects can be re-used multiple times.
* *
* Use ZSTD_initDStream() to start a new decompression operation, * Use ZSTD_initDStream() to start a new decompression operation,
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. * or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
@ -422,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/* ****************************************************************** /* ******************************************************************
* Advanced Streaming * Advanced Streaming functions
********************************************************************/ ********************************************************************/
/*====== compression ======*/ /*====== compression ======*/
@ -436,9 +436,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
/*====== decompression ======*/ /*====== decompression ======*/
/* advanced */ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);

23
tests/zstreamtest.c
View File

@ -212,6 +212,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic decompression test */ /* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
{ size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */
if (ZSTD_isError(r)) goto _output_error; }
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
@ -267,6 +269,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
} } } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
{ size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* Memory restriction */
DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
{ size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000); /* too small limit */
if (ZSTD_isError(r)) goto _output_error; }
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); }
_end: _end:
ZSTD_freeCStream(zc); ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd); ZSTD_freeDStream(zd);