From 17e482efdddf90c21b51fe4df545eed3dfe6b9ab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Aug 2016 16:58:10 +0200 Subject: [PATCH] added ZSTD_setDStreamParameter() --- lib/common/error_private.h | 1 + lib/common/error_public.h | 4 ++++ lib/decompress/zstd_decompress.c | 26 ++++++++++++++++++++------ lib/zstd.h | 10 ++++++---- tests/zstreamtest.c | 23 +++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 88906149..34c0c4c0 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 29050b3b..10f020cf 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -37,6 +37,9 @@ extern "C" { #endif +/*===== dependency =====*/ +#include /* size_t */ + /* **************************************** * error codes list @@ -45,6 +48,7 @@ typedef enum { ZSTD_error_no_error, ZSTD_error_GENERIC, ZSTD_error_prefix_unknown, + ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, ZSTD_error_compressionParameter_unsupported, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 524c2f56..bbeca0d8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -51,12 +51,12 @@ /*! -* STREAM_WINDOW_MAX : -* maximum window size accepted by DStream. -* frames requiring more memory will be rejected. +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. */ -#ifndef ZSTD_STREAM_WINDOW_MAX -# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */ #endif @@ -1329,6 +1329,7 @@ struct ZSTD_DStream_s { char* inBuff; size_t inBuffSize; size_t inPos; + size_t maxWindowSize; char* outBuff; size_t outBuffSize; size_t outStart; @@ -1362,6 +1363,7 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) zds->zd = ZSTD_createDCtx_advanced(customMem); if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; + zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; } @@ -1393,6 +1395,18 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + switch(paramType) + { + default : return ERROR(parameter_unknown); + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + } + return 0; +} + + size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; @@ -1451,11 +1465,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; - if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported); zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); diff --git a/lib/zstd.h b/lib/zstd.h index cb667484..fd6170c8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -87,7 +87,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * When `return==0`, data to decompress can have any size. * In which case, it's necessary to use streaming mode to decompress data. * Optionally, application may rely on its own implied limits. -* (For example, application own data could be necessarily cut into blocks <= 16 KB). +* (For example, application data could be necessarily cut into blocks <= 16 KB). * note 3 : decompressed size could be wrong or intentionally modified ! * Always ensure result fits within application's authorized limits ! * Each application can set its own limits. @@ -264,7 +264,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-init multiple times. +* ZSTD_DStream objects can be re-used multiple times. * * Use ZSTD_initDStream() to start a new decompression operation, * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. @@ -422,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Advanced Streaming +* Advanced Streaming functions ********************************************************************/ /*====== compression ======*/ @@ -436,9 +436,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /*====== decompression ======*/ -/* advanced */ +typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; + ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 4c81e8df..edfb07eb 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -212,6 +212,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */ + if (ZSTD_isError(r)) goto _output_error; } { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ @@ -267,6 +269,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++); + { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */ + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + /* Memory restriction */ + DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000); /* too small limit */ + if (ZSTD_isError(r)) goto _output_error; } + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ + DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); } + + _end: ZSTD_freeCStream(zc); ZSTD_freeDStream(zd);