diff --git a/lib/zbuff.c b/lib/zbuff.c index 4c1eb2cf..69341df8 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -298,28 +298,29 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) } - -/** ************************************************ -* Streaming decompression +/*-*************************************************************************** +* Streaming decompression howto * -* A ZBUFF_DCtx object is required to track streaming operation. +* A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) -* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; @@ -329,7 +330,7 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_dec #define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zc; - ZSTD_parameters params; + ZSTD_frameParams fParams; char* inBuff; size_t inBuffSize; size_t inPos; @@ -381,14 +382,16 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) /* *** Decompression *** */ -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) { const char* const istart = (const char*)src; const char* ip = istart; const char* const iend = istart + *srcSizePtr; char* const ostart = (char*)dst; char* op = ostart; - char* const oend = ostart + *maxDstSizePtr; + char* const oend = ostart + *dstCapacityPtr; U32 notDone = 1; while (notDone) { @@ -400,13 +403,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_readHeader : /* read header from src */ { - size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); zbc->hPos += *srcSizePtr; - *maxDstSizePtr = 0; + *dstCapacityPtr = 0; zbc->stage = ZBUFFds_loadHeader; return headerSize - zbc->hPos; } @@ -422,11 +425,11 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt src, *srcSizePtr); zbc->hPos += headerSize; ip += headerSize; - headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos); + headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ - *maxDstSizePtr = 0; + *dstCapacityPtr = 0; return headerSize - zbc->hPos; } // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ @@ -435,7 +438,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ { - size_t neededOutSize = (size_t)1 << zbc->params.windowLog; + size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog; size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); @@ -524,7 +527,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt } } *srcSizePtr = ip-istart; - *maxDstSizePtr = op-ostart; + *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); diff --git a/lib/zbuff.h b/lib/zbuff.h index d3275b7d..7a914a4a 100644 --- a/lib/zbuff.h +++ b/lib/zbuff.h @@ -75,7 +75,7 @@ ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstC ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); /*-************************************************* -* Streaming compression +* Streaming compression - howto * * A ZBUFF_CCtx object is required to track streaming operation. * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. @@ -127,26 +127,27 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, const void* src, size_t* srcSizePtr); /*-*************************************************************************** -* Streaming decompression +* Streaming decompression howto * * A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_decompressInit() to start a new decompression operation, * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be reused multiple times. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *dstCapacityPtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize() +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index fc88eb16..469410fa 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -103,7 +103,7 @@ struct ZSTD_CCtx_s size_t workSpaceSize; size_t blockSize; size_t hbSize; - char headerBuffer[ZSTD_frameHeaderSize_max]; + char headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -287,7 +287,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) } -/*! ZSTD_reduceIndex +/*! ZSTD_reduceIndex() : * rescale indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) @@ -308,6 +308,37 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, * Block entropic compression *********************************************************/ +/* Frame format description + Frame Header - [ Block Header - Block ] - Frame End + 1) Frame Header + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte - Frame Descriptor + 2) Block Header + - 3 bytes, starting with a 2-bits descriptor + Uncompressed, Compressed, Frame End, unused + 3) Block + See Block Format Description + 4) Frame End + - 3 bytes, compatible with Block Header +*/ + + +/* Frame descriptor + + 1 byte, using : + bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) + bit 4 : minmatch 4(0) or 3(1) + bit 5 : reserved (must be zero) + bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + + Optional : content size (0, 1, 2 or 8 bytes) + 0 : unknown + 1 : 0-255 bytes + 2 : 256 - 65535+256 + 8 : up to 16 exa +*/ + + /* Block format description Block = Literal Section - Sequences Section @@ -2099,10 +2130,10 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t /* Dictionary format : Magic == ZSTD_DICT_MAGIC (4 bytes) - Huff0 CTable (256 * 4 bytes) => to be changed to read from writeCTable + HUF_writeCTable(256) Dictionary content */ -/*! ZSTD_loadDictEntropyStats +/*! ZSTD_loadDictEntropyStats() : @return : size read from dictionary */ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { @@ -2160,7 +2191,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si } -/*! ZSTD_compressBegin_advanced +/*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, @@ -2173,6 +2204,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, errorCode = ZSTD_resetCCtx_advanced(zc, params); if (ZSTD_isError(errorCode)) return errorCode; + /* Write Frame Header */ MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */ ((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4)); zc->hbSize = ZSTD_frameHeaderSize_min; @@ -2195,8 +2227,8 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) } -/*! ZSTD_compressEnd -* Write frame epilogue +/*! ZSTD_compressEnd() : +* Write frame epilogue. * @return : nb of bytes written into dst (or an error code) */ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) { diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 60136b61..6e595228 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -94,13 +94,13 @@ typedef struct } blockProperties_t; -/* ******************************************************* +/*_******************************************************* * Memory operations **********************************************************/ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } -/* ************************************* +/*-************************************* * Error Management ***************************************/ unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } @@ -118,7 +118,7 @@ ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); } const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } -/* ************************************************************* +/*-************************************************************* * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, @@ -136,7 +136,7 @@ struct ZSTD_DCtx_s const void* dictEnd; size_t expected; size_t headerSize; - ZSTD_parameters params; + ZSTD_frameParams fParams; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagStaticTables; @@ -144,7 +144,7 @@ struct ZSTD_DCtx_s size_t litBufSize; size_t litSize; BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_frameHeaderSize_max]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } @@ -159,7 +159,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; - dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ + dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); return 0; } @@ -185,15 +185,15 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) } -/* ************************************************************* +/*-************************************************************* * Decompression section ***************************************************************/ /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_internal.h) - - 1 byte - Window Descriptor + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte - Frame Descriptor 2) Block Header - 3 bytes, starting with a 2-bits descriptor Uncompressed, Compressed, Frame End, unused @@ -203,7 +203,24 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) - 3 bytes, compatible with Block Header */ -/* Block format description + +/* Frame descriptor + + 1 byte, using : + bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) + bit 4 : minmatch 4(0) or 3(1) + bit 5 : reserved (must be zero) + bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + + Optional : content size (0, 1, 2 or 8 bytes) + 0 : unknown + 1 : 0-255 bytes + 2 : 256 - 65535+256 + 8 : up to 16 exa +*/ + + +/* Compressed Block, format description Block = Literal Section - Sequences Section Prerequisite : size of (compressed) block, maximum size of regenerated data @@ -269,47 +286,64 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) TO DO */ +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; -/** ZSTD_decodeFrameHeader_Part1() : -* decode the 1st part of the Frame Header, which tells Frame Header size. -* srcSize must be == ZSTD_frameHeaderSize_min. -* @return : the full size of the Frame Header */ -static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize) +/** ZSTD_frameHeaderSize() : +* srcSize must be >= ZSTD_frameHeaderSize_min. +* @return : size of the Frame Header */ +static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { - U32 magicNumber; - if (srcSize != ZSTD_frameHeaderSize_min) - return ERROR(srcSize_wrong); - magicNumber = MEM_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - zc->headerSize = ZSTD_frameHeaderSize_min; - return zc->headerSize; + U32 fcsId; + if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + fcsId = (((const BYTE*)src)[4]) >> 6; + return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } -size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) +/** ZSTD_getFrameParams() : +* decode Frame Header, or provide expected `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, not enough srcSize, provide expected `srcSize`, +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) { - U32 magicNumber; - if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max; + U32 magicNumber, fcsId; + const BYTE* ip = (const BYTE*)src; + BYTE frameDesc; + + if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; magicNumber = MEM_readLE32(src); if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - memset(params, 0, sizeof(*params)); - params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - params->searchLength = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH; - if ((((const BYTE*)src)[4] >> 5) != 0) return ERROR(frameParameter_unsupported); /* reserved 3 bits */ + + { size_t fhsize = ZSTD_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + frameDesc = ip[4]; + fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; + if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ + fcsId = frameDesc >> 6; + switch(fcsId) + { + default: /* impossible */ + case 0 : fparamsPtr->frameContentSize = 0; break; + case 1 : fparamsPtr->frameContentSize = ip[5]; break; + case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5); break; + case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; + } return 0; } -/** ZSTD_decodeFrameHeader_Part2() : -* decode the full Frame Header. + +/** ZSTD_decodeFrameHeader() : +* decode Frame Header. * srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1(). * @return : 0, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize) +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) { - size_t result; - if (srcSize != zc->headerSize) - return ERROR(srcSize_wrong); - result = ZSTD_getFrameParams(&(zc->params), src, srcSize); - if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + size_t result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); + if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); return result; } @@ -792,7 +826,7 @@ static size_t ZSTD_decompressSequences( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - const U32 mls = dctx->params.searchLength; + const U32 mls = dctx->fParams.mml; /* Build Decoding Tables */ errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, @@ -885,9 +919,9 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_continueDCtx -* dctx must have been properly initialized */ -static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, +/*! ZSTD_decompress_continueDCtx() : +* `dctx` must have been properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { @@ -901,7 +935,7 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, /* Frame Header */ { - size_t frameHeaderSize; + size_t frameHeaderSize, errorCode; if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) { @@ -910,12 +944,12 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); } #endif - frameHeaderSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); + frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + errorCode = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize); + if (ZSTD_isError(errorCode)) return errorCode; ip += frameHeaderSize; remainingSize -= frameHeaderSize; - frameHeaderSize = ZSTD_decodeFrameHeader_Part2(dctx, src, frameHeaderSize); - if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; } /* Loop on each block */ @@ -965,7 +999,7 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC { ZSTD_copyDCtx(dctx, refDCtx); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -977,7 +1011,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -986,6 +1020,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0); } + size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) @@ -1023,7 +1058,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co { /* get frame header size */ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); if (dctx->headerSize > ZSTD_frameHeaderSize_min) { @@ -1038,7 +1073,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co /* get frame header */ size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); - result = ZSTD_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); + result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 409e5e57..d67505ca 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -71,11 +71,9 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define BLOCKSIZE (128 KB) /* define, for static allocation */ +#define BLOCKSIZE (128 KB) /* define, for static allocation */ static const size_t ZSTD_blockHeaderSize = 3; -static const size_t ZSTD_frameHeaderSize_min = 5; -#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */ #define BIT7 128 #define BIT6 64 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 4f23fca9..f6eb8aa3 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -165,13 +165,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse ZSTD_CCtx to compress some new frame. */ +typedef struct { U64 frameContentSize; U32 windowLog; U32 mml; } ZSTD_frameParams; + +#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); - ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -182,15 +186,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. - First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). - This operation is independent, and just needs enough input data to properly decode the frame header. - Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding. - Result : 0 when successful, it means the ZSTD_parameters structure has been filled. - >0 : means there is not enough data into src. Provides the expected size to successfully decode header. + First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + It requires to read the beginning of compressed frame. + The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max. + If you don't provide enough length, function will return the minimum size it wants to produce a result. + Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. + >0 : means there is not enough data into src. Provides the expected size to successfully decode header. errorCode, which can be tested using ZSTD_isError() - Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict() - Alternatively, you can copy a prepared context, using ZSTD_copyDCtx() + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().