facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #149 from Cyan4973/framesize

Browse Source 
Framesize
This commit is contained in:
Yann Collet 2016-03-15 00:33:40 +01:00
commit 5b31fa6f16
6 changed files with 175 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
lib/zbuff.c
View File

@ -298,28 +298,29 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
} }
/*-***************************************************************************
/** ************************************************ * Streaming decompression howto
* Streaming decompression
* *
* A ZBUFF_DCtx object is required to track streaming operation. * A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation. * Use ZBUFF_decompressInit() to start a new decompression operation,
* ZBUFF_DCtx objects can be reused multiple times. * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFF_DCtx objects can be re-init multiple times.
* *
* Use ZBUFF_decompressContinue() repetitively to consume your input. * Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size. * *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . * The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
* or 0 when a frame is completely decoded * or 0 when a frame is completely decoded,
* or an error code, which can be tested using ZBUFF_isError(). * or an error code, which can be tested using ZBUFF_isError().
* *
* Hint : recommended buffer sizes (not compulsory) * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * input : ZBUFF_recommendedDInSize == 128KB + 3;
* **************************************************/ * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
@ -329,7 +330,7 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_dec
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ #define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
struct ZBUFF_DCtx_s { struct ZBUFF_DCtx_s {
ZSTD_DCtx* zc; ZSTD_DCtx* zc;
ZSTD_parameters params; ZSTD_frameParams fParams;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inPos; size_t inPos;
@ -381,14 +382,16 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
/* *** Decompression *** */ /* *** Decompression *** */
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{ {
const char* const istart = (const char*)src; const char* const istart = (const char*)src;
const char* ip = istart; const char* ip = istart;
const char* const iend = istart + *srcSizePtr; const char* const iend = istart + *srcSizePtr;
char* const ostart = (char*)dst; char* const ostart = (char*)dst;
char* op = ostart; char* op = ostart;
char* const oend = ostart + *maxDstSizePtr; char* const oend = ostart + *dstCapacityPtr;
U32 notDone = 1; U32 notDone = 1;
while (notDone) { while (notDone) {
@ -400,13 +403,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
case ZBUFFds_readHeader : case ZBUFFds_readHeader :
/* read header from src */ /* read header from src */
{ {
size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) { if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */ /* not enough input to decode header : tell how many bytes would be necessary */
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
zbc->hPos += *srcSizePtr; zbc->hPos += *srcSizePtr;
*maxDstSizePtr = 0; *dstCapacityPtr = 0;
zbc->stage = ZBUFFds_loadHeader; zbc->stage = ZBUFFds_loadHeader;
return headerSize - zbc->hPos; return headerSize - zbc->hPos;
} }
@ -422,11 +425,11 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
src, *srcSizePtr); src, *srcSizePtr);
zbc->hPos += headerSize; zbc->hPos += headerSize;
ip += headerSize; ip += headerSize;
headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos); headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) { if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */ /* not enough input to decode header : tell how many bytes would be necessary */
*maxDstSizePtr = 0; *dstCapacityPtr = 0;
return headerSize - zbc->hPos; return headerSize - zbc->hPos;
} }
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
@ -435,7 +438,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
case ZBUFFds_decodeHeader: case ZBUFFds_decodeHeader:
/* apply header to create / resize buffers */ /* apply header to create / resize buffers */
{ {
size_t neededOutSize = (size_t)1 << zbc->params.windowLog; size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog;
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) { if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff); free(zbc->inBuff);
@ -524,7 +527,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
} } } }
*srcSizePtr = ip-istart; *srcSizePtr = ip-istart;
*maxDstSizePtr = op-ostart; *dstCapacityPtr = op-ostart;
{ {
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);

17
lib/zbuff.h
View File

@ -75,7 +75,7 @@ ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstC
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/*-************************************************* /*-*************************************************
* Streaming compression * Streaming compression - howto
* *
* A ZBUFF_CCtx object is required to track streaming operation. * A ZBUFF_CCtx object is required to track streaming operation.
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
@ -127,26 +127,27 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
const void* src, size_t* srcSizePtr); const void* src, size_t* srcSizePtr);
/*-*************************************************************************** /*-***************************************************************************
* Streaming decompression * Streaming decompression howto
* *
* A ZBUFF_DCtx object is required to track streaming operations. * A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation, * Use ZBUFF_decompressInit() to start a new decompression operation,
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFF_DCtx objects can be reused multiple times. * Note that ZBUFF_DCtx objects can be re-init multiple times.
* *
* Use ZBUFF_decompressContinue() repetitively to consume your input. * Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size. * *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. * The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
* or 0 when a frame is completely decoded * or 0 when a frame is completely decoded,
* or an error code, which can be tested using ZBUFF_isError(). * or an error code, which can be tested using ZBUFF_isError().
* *
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize() * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * input : ZBUFF_recommendedDInSize == 128KB + 3;
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/ * *******************************************************************************/

46
lib/zstd_compress.c
View File

@ -103,7 +103,7 @@ struct ZSTD_CCtx_s
size_t workSpaceSize; size_t workSpaceSize;
size_t blockSize; size_t blockSize;
size_t hbSize; size_t hbSize;
char headerBuffer[ZSTD_frameHeaderSize_max]; char headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
seqStore_t seqStore; /* sequences storage ptrs */ seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable; U32* hashTable;
@ -287,7 +287,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
} }
/*! ZSTD_reduceIndex /*! ZSTD_reduceIndex() :
* rescale indexes to avoid future overflow (indexes are U32) */ * rescale indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_CCtx* zc, static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
const U32 reducerValue) const U32 reducerValue)
@ -308,6 +308,37 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
* Block entropic compression * Block entropic compression
*********************************************************/ *********************************************************/
/* Frame format description
Frame Header - [ Block Header - Block ] - Frame End
1) Frame Header
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
- 1 byte - Frame Descriptor
2) Block Header
- 3 bytes, starting with a 2-bits descriptor
Uncompressed, Compressed, Frame End, unused
3) Block
See Block Format Description
4) Frame End
- 3 bytes, compatible with Block Header
*/
/* Frame descriptor
1 byte, using :
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
bit 4 : minmatch 4(0) or 3(1)
bit 5 : reserved (must be zero)
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
Optional : content size (0, 1, 2 or 8 bytes)
0 : unknown
1 : 0-255 bytes
2 : 256 - 65535+256
8 : up to 16 exa
*/
/* Block format description /* Block format description
Block = Literal Section - Sequences Section Block = Literal Section - Sequences Section
@ -2099,10 +2130,10 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
/* Dictionary format : /* Dictionary format :
Magic == ZSTD_DICT_MAGIC (4 bytes) Magic == ZSTD_DICT_MAGIC (4 bytes)
Huff0 CTable (256 * 4 bytes) => to be changed to read from writeCTable HUF_writeCTable(256)
Dictionary content Dictionary content
*/ */
/*! ZSTD_loadDictEntropyStats /*! ZSTD_loadDictEntropyStats() :
@return : size read from dictionary */ @return : size read from dictionary */
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{ {
@ -2160,7 +2191,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
} }
/*! ZSTD_compressBegin_advanced /*! ZSTD_compressBegin_advanced() :
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
@ -2173,6 +2204,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
errorCode = ZSTD_resetCCtx_advanced(zc, params); errorCode = ZSTD_resetCCtx_advanced(zc, params);
if (ZSTD_isError(errorCode)) return errorCode; if (ZSTD_isError(errorCode)) return errorCode;
/* Write Frame Header */
MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */ MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */
((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4)); ((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4));
zc->hbSize = ZSTD_frameHeaderSize_min; zc->hbSize = ZSTD_frameHeaderSize_min;
@ -2195,8 +2227,8 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
} }
/*! ZSTD_compressEnd /*! ZSTD_compressEnd() :
* Write frame epilogue * Write frame epilogue.
* @return : nb of bytes written into dst (or an error code) */ * @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize)
{ {

133
lib/zstd_decompress.c
View File

@ -94,13 +94,13 @@ typedef struct
} blockProperties_t; } blockProperties_t;
/* ******************************************************* /*_*******************************************************
* Memory operations * Memory operations
**********************************************************/ **********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
/* ************************************* /*-*************************************
* Error Management * Error Management
***************************************/ ***************************************/
unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
@ -118,7 +118,7 @@ ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); }
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
/* ************************************************************* /*-*************************************************************
* Context management * Context management
***************************************************************/ ***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
const void* dictEnd; const void* dictEnd;
size_t expected; size_t expected;
size_t headerSize; size_t headerSize;
ZSTD_parameters params; ZSTD_frameParams fParams;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage; ZSTD_dStage stage;
U32 flagStaticTables; U32 flagStaticTables;
@ -144,7 +144,7 @@ struct ZSTD_DCtx_s
size_t litBufSize; size_t litBufSize;
size_t litSize; size_t litSize;
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_frameHeaderSize_max]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
@ -159,7 +159,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->dictEnd = NULL; dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog; dctx->hufTableX4[0] = HufLog;
dctx->flagStaticTables = 0; dctx->flagStaticTables = 0;
dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength);
return 0; return 0;
} }
@ -185,15 +185,15 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
} }
/* ************************************************************* /*-*************************************************************
* Decompression section * Decompression section
***************************************************************/ ***************************************************************/
/* Frame format description /* Frame format description
Frame Header - [ Block Header - Block ] - Frame End Frame Header - [ Block Header - Block ] - Frame End
1) Frame Header 1) Frame Header
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_internal.h) - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
- 1 byte - Window Descriptor - 1 byte - Frame Descriptor
2) Block Header 2) Block Header
- 3 bytes, starting with a 2-bits descriptor - 3 bytes, starting with a 2-bits descriptor
Uncompressed, Compressed, Frame End, unused Uncompressed, Compressed, Frame End, unused
@ -203,7 +203,24 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
- 3 bytes, compatible with Block Header - 3 bytes, compatible with Block Header
*/ */
/* Block format description
/* Frame descriptor
1 byte, using :
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
bit 4 : minmatch 4(0) or 3(1)
bit 5 : reserved (must be zero)
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
Optional : content size (0, 1, 2 or 8 bytes)
0 : unknown
1 : 0-255 bytes
2 : 256 - 65535+256
8 : up to 16 exa
*/
/* Compressed Block, format description
Block = Literal Section - Sequences Section Block = Literal Section - Sequences Section
Prerequisite : size of (compressed) block, maximum size of regenerated data Prerequisite : size of (compressed) block, maximum size of regenerated data
@ -269,47 +286,64 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
TO DO TO DO
*/ */
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
/** ZSTD_decodeFrameHeader_Part1() : /** ZSTD_frameHeaderSize() :
* decode the 1st part of the Frame Header, which tells Frame Header size. * srcSize must be >= ZSTD_frameHeaderSize_min.
* srcSize must be == ZSTD_frameHeaderSize_min. * @return : size of the Frame Header */
* @return : the full size of the Frame Header */ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize)
{ {
U32 magicNumber; U32 fcsId;
if (srcSize != ZSTD_frameHeaderSize_min) if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
return ERROR(srcSize_wrong); fcsId = (((const BYTE*)src)[4]) >> 6;
magicNumber = MEM_readLE32(src); return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
zc->headerSize = ZSTD_frameHeaderSize_min;
return zc->headerSize;
} }
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) /** ZSTD_getFrameParams() :
* decode Frame Header, or provide expected `srcSize`.
* @return : 0, `fparamsPtr` is correctly filled,
* >0, not enough srcSize, provide expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
{ {
U32 magicNumber; U32 magicNumber, fcsId;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max; const BYTE* ip = (const BYTE*)src;
BYTE frameDesc;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
magicNumber = MEM_readLE32(src); magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
memset(params, 0, sizeof(*params));
params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN; { size_t fhsize = ZSTD_frameHeaderSize(src, srcSize);
params->searchLength = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH; if (srcSize < fhsize) return fhsize; }
if ((((const BYTE*)src)[4] >> 5) != 0) return ERROR(frameParameter_unsupported); /* reserved 3 bits */
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
frameDesc = ip[4];
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH;
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
fcsId = frameDesc >> 6;
switch(fcsId)
{
default: /* impossible */
case 0 : fparamsPtr->frameContentSize = 0; break;
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5); break;
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
}
return 0; return 0;
} }
/** ZSTD_decodeFrameHeader_Part2() :
* decode the full Frame Header. /** ZSTD_decodeFrameHeader() :
* decode Frame Header.
* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1(). * srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1().
* @return : 0, or an error code, which can be tested using ZSTD_isError() */ * @return : 0, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
{ {
size_t result; size_t result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
if (srcSize != zc->headerSize) if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
return ERROR(srcSize_wrong);
result = ZSTD_getFrameParams(&(zc->params), src, srcSize);
if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
return result; return result;
} }
@ -792,7 +826,7 @@ static size_t ZSTD_decompressSequences(
const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
const U32 mls = dctx->params.searchLength; const U32 mls = dctx->fParams.mml;
/* Build Decoding Tables */ /* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
@ -885,9 +919,9 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
} }
/*! ZSTD_decompress_continueDCtx /*! ZSTD_decompress_continueDCtx() :
* dctx must have been properly initialized */ * `dctx` must have been properly initialized */
static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
@ -901,7 +935,7 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx,
/* Frame Header */ /* Frame Header */
{ {
size_t frameHeaderSize; size_t frameHeaderSize, errorCode;
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
{ {
@ -910,12 +944,12 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx,
return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
} }
#endif #endif
frameHeaderSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
errorCode = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize);
if (ZSTD_isError(errorCode)) return errorCode;
ip += frameHeaderSize; remainingSize -= frameHeaderSize; ip += frameHeaderSize; remainingSize -= frameHeaderSize;
frameHeaderSize = ZSTD_decodeFrameHeader_Part2(dctx, src, frameHeaderSize);
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
} }
/* Loop on each block */ /* Loop on each block */
@ -965,7 +999,7 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC
{ {
ZSTD_copyDCtx(dctx, refDCtx); ZSTD_copyDCtx(dctx, refDCtx);
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize);
} }
@ -977,7 +1011,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength);
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize);
} }
@ -986,6 +1020,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const
return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0); return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0);
} }
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
@ -1023,7 +1058,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
{ {
/* get frame header size */ /* get frame header size */
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
dctx->headerSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) { if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
@ -1038,7 +1073,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
/* get frame header */ /* get frame header */
size_t result; size_t result;
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
result = ZSTD_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result)) return result;
dctx->expected = ZSTD_blockHeaderSize; dctx->expected = ZSTD_blockHeaderSize;
dctx->stage = ZSTDds_decodeBlockHeader; dctx->stage = ZSTDds_decodeBlockHeader;

4
lib/zstd_internal.h
View File

@ -71,11 +71,9 @@
#define MB *(1 <<20) #define MB *(1 <<20)
#define GB *(1U<<30) #define GB *(1U<<30)
#define BLOCKSIZE (128 KB) /* define, for static allocation */ #define BLOCKSIZE (128 KB) /* define, for static allocation */
static const size_t ZSTD_blockHeaderSize = 3; static const size_t ZSTD_blockHeaderSize = 3;
static const size_t ZSTD_frameHeaderSize_min = 5;
#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */
#define BIT7 128 #define BIT7 128
#define BIT6 64 #define BIT6 64

23
lib/zstd_static.h
View File

@ -165,13 +165,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
You can then reuse ZSTD_CCtx to compress some new frame. You can then reuse ZSTD_CCtx to compress some new frame.
*/ */
typedef struct { U64 frameContentSize; U32 windowLog; U32 mml; } ZSTD_frameParams;
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_min = 5;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@ -182,15 +186,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams().
This operation is independent, and just needs enough input data to properly decode the frame header. It requires to read the beginning of compressed frame.
Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding. The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max.
Result : 0 when successful, it means the ZSTD_parameters structure has been filled. If you don't provide enough length, function will return the minimum size it wants to produce a result.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header. Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError() errorCode, which can be tested using ZSTD_isError()
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict() Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx() Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().