changed `BLOCKSIZE` into `ZSTD_BLOCKSIZE_MAX`
Yann Collet
parent
69afc716d3
commit
569b81adb1
59
lib/zbuff.c
59
lib/zbuff.c
|
|
@ -40,15 +40,16 @@
|
|||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static size_t ZBUFF_blockHeaderSize = 3;
|
||||
static size_t ZBUFF_endFrameSize = 3;
|
||||
static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
|
||||
|
||||
|
||||
/*_**************************************************
|
||||
* Streaming compression
|
||||
|
|
@ -59,28 +60,28 @@ static size_t ZBUFF_endFrameSize = 3;
|
|||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
|
||||
* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
|
||||
* Note that it will not output more than *maxDstSizePtr.
|
||||
* Note that it will not output more than *dstCapacityPtr.
|
||||
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory)
|
||||
* input : 128 KB block size is the internal unit, it improves latency to use this value.
|
||||
* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed.
|
||||
* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
|
||||
* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
|
||||
* **************************************************/
|
||||
|
||||
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
|
||||
|
|
@ -137,7 +138,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dic
|
|||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize);
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
free(zbc->outBuff); /* should not be necessary */
|
||||
|
|
@ -178,7 +179,7 @@ static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, si
|
|||
}
|
||||
|
||||
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* maxDstSizePtr,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr,
|
||||
int flush) /* aggregate : wait for full block before compressing */
|
||||
{
|
||||
|
|
@ -188,7 +189,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
|||
const char* const iend = istart + *srcSizePtr;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *maxDstSizePtr;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
|
||||
while (notDone) {
|
||||
switch(zbc->stage)
|
||||
|
|
@ -248,7 +249,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
|||
}
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*maxDstSizePtr = op - ostart;
|
||||
*dstCapacityPtr = op - ostart;
|
||||
{
|
||||
size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zbc->blockSize;
|
||||
|
|
@ -257,30 +258,30 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
|||
}
|
||||
|
||||
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* maxDstSizePtr,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0);
|
||||
return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Finalize *** */
|
||||
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
size_t srcSize = 0;
|
||||
ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */
|
||||
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + *maxDstSizePtr;
|
||||
size_t outSize = *maxDstSizePtr;
|
||||
BYTE* const oend = ostart + *dstCapacityPtr;
|
||||
size_t outSize = *dstCapacityPtr;
|
||||
size_t epilogueSize, remaining;
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
op += outSize;
|
||||
|
|
@ -291,7 +292,7 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
|||
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
|
||||
op += outSize;
|
||||
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
|
||||
*maxDstSizePtr = op-ostart; /* tells how many bytes were written */
|
||||
*dstCapacityPtr = op-ostart; /* tells how many bytes were written */
|
||||
return remaining;
|
||||
}
|
||||
|
||||
|
|
@ -406,7 +407,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
|||
} }
|
||||
|
||||
/* Frame header provides buffer sizes */
|
||||
{ size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
|
||||
{ size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
free(zbc->inBuff);
|
||||
zbc->inBuffSize = neededInSize;
|
||||
|
|
@ -476,7 +477,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
|||
zbc->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) {
|
||||
zbc->stage = ZBUFFds_read;
|
||||
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
|
||||
if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize)
|
||||
zbc->outStart = zbc->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
|
|
@ -492,7 +493,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
|||
*dstCapacityPtr = op-ostart;
|
||||
{
|
||||
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get following block header too */
|
||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
|
|
@ -506,7 +507,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
|||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
size_t ZBUFF_recommendedCInSize(void) { return BLOCKSIZE; }
|
||||
size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(BLOCKSIZE) + ZBUFF_blockHeaderSize + ZBUFF_endFrameSize; }
|
||||
size_t ZBUFF_recommendedDInSize(void) { return BLOCKSIZE + ZBUFF_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFF_recommendedDOutSize(void) { return BLOCKSIZE; }
|
||||
size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
||||
size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
|
||||
size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
||||
|
|
|
|||
|
|
@ -587,7 +587,7 @@ typedef struct
|
|||
{
|
||||
ZSTD_CCtx* ref;
|
||||
ZSTD_CCtx* zc;
|
||||
void* workPlace; /* must be BLOCKSIZE allocated */
|
||||
void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
|
||||
} EStats_ress_t;
|
||||
|
||||
|
||||
|
|
@ -599,9 +599,9 @@ static void ZDICT_countEStats(EStats_ress_t esr,
|
|||
const U32* u32Ptr;
|
||||
seqStore_t seqStore;
|
||||
|
||||
if (srcSize > BLOCKSIZE) srcSize = BLOCKSIZE; /* protection vs large samples */
|
||||
if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */
|
||||
ZSTD_copyCCtx(esr.zc, esr.ref);
|
||||
ZSTD_compressBlock(esr.zc, esr.workPlace, BLOCKSIZE, src, srcSize);
|
||||
ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
||||
seqStore = ZSTD_copySeqStore(esr.zc);
|
||||
|
||||
/* count stats */
|
||||
|
|
@ -654,7 +654,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|||
for (u=0; u<=MaxLL; u++) litlengthCount[u]=1;
|
||||
esr.ref = ZSTD_createCCtx();
|
||||
esr.zc = ZSTD_createCCtx();
|
||||
esr.workPlace = malloc(BLOCKSIZE);
|
||||
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
|
||||
if (!esr.ref || !esr.zc || !esr.workPlace) {
|
||||
eSize = ERROR(memory_allocation);
|
||||
DISPLAYLEVEL(1, "Not enough memory");
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ void ZSTD_validateParams(ZSTD_parameters* params)
|
|||
|
||||
size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for paramagrill */
|
||||
{ /* copy / pasted from ZSTD_resetCCtx_advanced */
|
||||
const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
|
||||
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog);
|
||||
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
|
||||
const U32 divider = (params.searchLength==3) ? 3 : 4;
|
||||
const size_t maxNbSeq = blockSize / divider;
|
||||
|
|
@ -186,7 +186,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for parama
|
|||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
ZSTD_parameters params)
|
||||
{ /* note : params considered validated here */
|
||||
const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
|
||||
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog);
|
||||
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
|
||||
const U32 divider = (params.searchLength==3) ? 3 : 4;
|
||||
const size_t maxNbSeq = blockSize / divider;
|
||||
|
|
@ -2079,7 +2079,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
|
|||
|
||||
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong);
|
||||
if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
||||
zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength);
|
||||
return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0);
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ struct ZSTD_DCtx_s
|
|||
const BYTE* litPtr;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
|
||||
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
|
||||
|
||||
|
|
@ -181,7 +181,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
|||
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
{
|
||||
memcpy(dstDCtx, srcDCtx,
|
||||
sizeof(ZSTD_DCtx) - (BLOCKSIZE+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */
|
||||
sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -411,7 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||
litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4];
|
||||
break;
|
||||
}
|
||||
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
||||
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
||||
|
||||
if (HUF_isError(singleStream ?
|
||||
HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
|
|
@ -419,7 +419,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||
return ERROR(corruption_detected);
|
||||
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+8;
|
||||
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8;
|
||||
dctx->litSize = litSize;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
|
|
@ -442,7 +442,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
|
||||
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
|
||||
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
|
|
@ -468,7 +468,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
|
||||
memcpy(dctx->litBuffer, istart+lhSize, litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+8;
|
||||
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
|
|
@ -495,10 +495,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||
litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
|
||||
break;
|
||||
}
|
||||
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
||||
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
||||
memset(dctx->litBuffer, istart[lhSize], litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
|
||||
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+1;
|
||||
}
|
||||
|
|
@ -892,7 +892,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
|||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t litCSize;
|
||||
|
||||
if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong);
|
||||
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
||||
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- zstd homepage : https://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_CCOMMON_H_MODULE
|
||||
#define ZSTD_CCOMMON_H_MODULE
|
||||
|
|
@ -71,9 +71,8 @@
|
|||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BLOCKSIZE (128 KB) /* define, for static allocation */
|
||||
|
||||
static const size_t ZSTD_blockHeaderSize = 3;
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
|
||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
|||
|
||||
/*! ZSTD_getParams() :
|
||||
* @return ZSTD_parameters structure for a selected compression level and srcSize.
|
||||
* `srcSizeHint` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize);
|
||||
|
||||
/*! ZSTD_validateParams() :
|
||||
* correct params value to remain within authorized range */
|
||||
|
|
@ -112,7 +112,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
|||
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
||||
* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the compression operation */
|
||||
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
|
||||
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
|
|
@ -124,19 +124,19 @@ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
|
|||
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
|
||||
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Streaming functions (direct mode)
|
||||
****************************************/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
|
@ -187,11 +187,11 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
|||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams().
|
||||
It can provide the minimum size of buffer required to properly decompress data,
|
||||
It can provide the minimum size of rolling buffer required to properly decompress data,
|
||||
and optionally the final size of uncompressed content.
|
||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||
It is done by reading a certain amount of the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max.
|
||||
Frame parameters are extracted from the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
|
||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||
|
|
@ -206,7 +206,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
|||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
||||
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
||||
|
||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
|
||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
|
|
@ -218,10 +218,10 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
|||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
User will have to take in charge required information to regenerate data, such as block sizes.
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Uncompressed block size must be <= 128 KB
|
||||
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
|
||||
- Compressing or decompressing requires a context structure
|
||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
|
|
@ -235,6 +235,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
|||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
|
||||
*/
|
||||
|
||||
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@
|
|||
#define DEFAULT_CHUNKSIZE (4<<20)
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t sampleSize = 10000000;
|
||||
static const size_t g_sampleSize = 10000000;
|
||||
|
||||
|
||||
/*_************************************
|
||||
|
|
@ -256,7 +256,7 @@ size_t local_ZBUFF_compress(void* dst, size_t dstSize, void* buff2, const void*
|
|||
}
|
||||
|
||||
static ZBUFF_DCtx* g_zbdc = NULL;
|
||||
size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
static size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t srcRead = g_cSize, dstWritten = dstSize;
|
||||
(void)src; (void)srcSize;
|
||||
|
|
@ -270,7 +270,7 @@ size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void
|
|||
/*_*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
|
||||
{
|
||||
BYTE* dstBuff;
|
||||
size_t dstBuffSize;
|
||||
|
|
@ -405,9 +405,9 @@ _cleanOut:
|
|||
}
|
||||
|
||||
|
||||
int benchSample(U32 benchNb)
|
||||
static int benchSample(U32 benchNb)
|
||||
{
|
||||
size_t const benchedSize = sampleSize;
|
||||
size_t const benchedSize = g_sampleSize;
|
||||
const char* name = "Sample 10MiB";
|
||||
|
||||
/* Allocation */
|
||||
|
|
@ -430,16 +430,15 @@ int benchSample(U32 benchNb)
|
|||
}
|
||||
|
||||
|
||||
int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb)
|
||||
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
|
||||
{
|
||||
/* Loop for each file */
|
||||
int fileIdx=0;
|
||||
while (fileIdx<nbFiles) {
|
||||
const char* inFileName = fileNamesTable[fileIdx++];
|
||||
int fileIdx;
|
||||
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
|
||||
const char* inFileName = fileNamesTable[fileIdx];
|
||||
FILE* inFile = fopen( inFileName, "rb" );
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
size_t readSize;
|
||||
void* origBuff;
|
||||
|
||||
/* Check file existence */
|
||||
|
|
@ -450,7 +449,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20));
|
||||
|
||||
/* Alloc */
|
||||
origBuff = malloc(benchedSize);
|
||||
|
|
@ -458,14 +457,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
|
||||
/* Fill input buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
||||
if(readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
}
|
||||
{
|
||||
size_t readSize = fread(origBuff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
if (readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(origBuff);
|
||||
return 13;
|
||||
} }
|
||||
|
||||
/* bench */
|
||||
DISPLAY("\r%79s\r", "");
|
||||
|
|
@ -474,6 +473,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, U32 benchNb)
|
|||
benchMem(origBuff, benchedSize, benchNb);
|
||||
else
|
||||
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
|
||||
|
||||
free(origBuff);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -489,8 +490,9 @@ static int usage(const char* exename)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(void)
|
||||
static int usage_advanced(const char* exename)
|
||||
{
|
||||
usage(exename);
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -b# : test only function # \n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
|
|
@ -502,41 +504,34 @@ static int badusage(const char* exename)
|
|||
{
|
||||
DISPLAY("Wrong parameters\n");
|
||||
usage(exename);
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i,
|
||||
filenamesStart=0,
|
||||
result;
|
||||
const char* exename=argv[0];
|
||||
const char* input_filename=0;
|
||||
int i, filenamesStart=0, result;
|
||||
const char* exename = argv[0];
|
||||
const char* input_filename = NULL;
|
||||
U32 benchNb = 0, main_pause = 0;
|
||||
|
||||
// Welcome message
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
if (argc<1) return badusage(exename);
|
||||
|
||||
if (argc<1) { badusage(exename); return 1; }
|
||||
|
||||
for(i=1; i<argc; i++)
|
||||
{
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
/* Commands (note : aggregated commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
while (argument[1]!=0)
|
||||
{
|
||||
argument ++;
|
||||
while (argument[1]!=0) {
|
||||
argument++;
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
/* Display help on usage */
|
||||
case 'h' :
|
||||
case 'H': usage(exename); usage_advanced(); return 0;
|
||||
case 'H': return usage_advanced(exename);
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause = 1; break;
|
||||
|
|
@ -560,7 +555,7 @@ int main(int argc, const char** argv)
|
|||
}
|
||||
break;
|
||||
|
||||
/* Select specific algorithm to bench */
|
||||
/* Select compressibility of synthetic sample */
|
||||
case 'P':
|
||||
{ U32 proba32 = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
|
|
@ -573,7 +568,7 @@ int main(int argc, const char** argv)
|
|||
break;
|
||||
|
||||
/* Unknown command */
|
||||
default : badusage(exename); return 1;
|
||||
default : return badusage(exename);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
|
|
@ -583,9 +578,10 @@ int main(int argc, const char** argv)
|
|||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
}
|
||||
|
||||
if (filenamesStart==0)
|
||||
if (filenamesStart==0) /* no input file */
|
||||
result = benchSample(benchNb);
|
||||
else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
|
||||
else
|
||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
|
||||
|
||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue