Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
Przemyslaw Skibinski
commit
b876b96ce1
7
NEWS
7
NEWS
|
|
@ -1,3 +1,10 @@
|
|||
v1.1.4
|
||||
cli : new : advanced comnmand --priority=rt, by Przemyslaw Skibinski
|
||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||
API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
|
||||
API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size, by Sean Purcell
|
||||
doc : new : educational decoder, by Sean Purcell
|
||||
|
||||
v1.1.3
|
||||
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
|
||||
cli : new : experimental target `make zstdmt`, with multi-threading support
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ struct Options {
|
|||
|
||||
ZSTD_parameters determineParameters() const {
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
params.fParams.checksumFlag = checksum;
|
||||
if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
|
||||
params.cParams.windowLog = maxWindowLog;
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
|
|||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = loadFile_orDie(fname, &cSize);
|
||||
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
|
||||
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
|
||||
if (rSize==0) {
|
||||
fprintf(stderr, "%s : original size unknown \n", fname);
|
||||
exit(6);
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ static void decompress(const char* fname)
|
|||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = loadFile_X(fname, &cSize);
|
||||
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
|
||||
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
|
||||
if (rSize==0) {
|
||||
printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
|
||||
exit(5);
|
||||
|
|
|
|||
|
|
@ -344,8 +344,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
|
|||
{
|
||||
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
|
||||
|
||||
|
||||
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
|
||||
{ ZSTD_parameters params = srcCCtx->params;
|
||||
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
|
||||
}
|
||||
|
||||
/* copy tables */
|
||||
{ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
|
|
@ -2362,7 +2366,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
|
||||
|
|
@ -2845,7 +2849,11 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
|
|||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
|
||||
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
|
||||
else {
|
||||
ZSTD_parameters params = cdict->refContext->params;
|
||||
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -2939,7 +2947,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
|
|||
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
|
||||
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
|
||||
|
||||
|
|
@ -2957,6 +2965,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
|||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
|
||||
zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
|
||||
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
|
|
@ -2988,7 +3004,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
|||
zcs->checksum = params.fParams.checksumFlag > 0;
|
||||
zcs->params = params;
|
||||
|
||||
return ZSTD_resetCStream(zcs, pledgedSrcSize);
|
||||
return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
|
||||
}
|
||||
|
||||
/* note : cdict must outlive compression session */
|
||||
|
|
|
|||
|
|
@ -306,6 +306,86 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|||
return 0;
|
||||
}
|
||||
|
||||
/** ZSTD_getFrameContentSize() :
|
||||
* compatible with legacy mode
|
||||
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
|
||||
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
|
||||
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) {
|
||||
unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
|
||||
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
|
||||
}
|
||||
#endif
|
||||
{
|
||||
ZSTD_frameParams fParams;
|
||||
if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
|
||||
if (fParams.windowSize == 0) {
|
||||
/* Either skippable or empty frame, size == 0 either way */
|
||||
return 0;
|
||||
} else if (fParams.frameContentSize != 0) {
|
||||
return fParams.frameContentSize;
|
||||
} else {
|
||||
return ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_findDecompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* `srcSize` must be the exact length of some number of ZSTD compressed and/or
|
||||
* skippable frames
|
||||
* @return : decompressed size of the frames contained */
|
||||
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
{
|
||||
unsigned long long totalDstSize = 0;
|
||||
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
||||
const U32 magicNumber = MEM_readLE32(src);
|
||||
|
||||
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
size_t skippableSize;
|
||||
if (srcSize < ZSTD_skippableHeaderSize)
|
||||
return ERROR(srcSize_wrong);
|
||||
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
|
||||
ZSTD_skippableHeaderSize;
|
||||
if (srcSize < skippableSize) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
|
||||
src = (const BYTE *)src + skippableSize;
|
||||
srcSize -= skippableSize;
|
||||
continue;
|
||||
}
|
||||
|
||||
{
|
||||
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
||||
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
|
||||
|
||||
/* check for overflow */
|
||||
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
|
||||
totalDstSize += ret;
|
||||
}
|
||||
{
|
||||
size_t const frameSrcSize = ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
if (ZSTD_isError(frameSrcSize)) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
|
||||
src = (const BYTE *)src + frameSrcSize;
|
||||
srcSize -= frameSrcSize;
|
||||
}
|
||||
}
|
||||
|
||||
if (srcSize) {
|
||||
return ZSTD_CONTENTSIZE_ERROR;
|
||||
}
|
||||
|
||||
return totalDstSize;
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_getDecompressedSize() :
|
||||
* compatible with legacy mode
|
||||
|
|
@ -316,14 +396,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
|||
- frame header not complete (`srcSize` too small) */
|
||||
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize);
|
||||
#endif
|
||||
{ ZSTD_frameParams fparams;
|
||||
size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize);
|
||||
if (frResult!=0) return 0;
|
||||
return fparams.frameContentSize;
|
||||
}
|
||||
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
||||
return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1363,27 +1437,79 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
|
|||
return length;
|
||||
}
|
||||
|
||||
/** ZSTD_getFrameCompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* `src` must point to the start of a ZSTD or ZSTD legacy frame
|
||||
* `srcSize` must be at least as large as the frame contained
|
||||
* @return : the compressed size of the frame starting at `src` */
|
||||
size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
||||
#endif
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const ipstart = ip;
|
||||
size_t remainingSize = srcSize;
|
||||
ZSTD_frameParams fParams;
|
||||
|
||||
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
|
||||
/* Frame Header */
|
||||
{
|
||||
size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
|
||||
if (ZSTD_isError(ret)) return ret;
|
||||
if (ret > 0) return ERROR(srcSize_wrong);
|
||||
}
|
||||
|
||||
ip += headerSize;
|
||||
remainingSize -= headerSize;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
blockProperties_t blockProperties;
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
ip += ZSTD_blockHeaderSize + cBlockSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
|
||||
|
||||
if (blockProperties.lastBlock) break;
|
||||
}
|
||||
|
||||
if (fParams.checksumFlag) { /* Frame content checksum */
|
||||
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
||||
ip += 4;
|
||||
remainingSize -= 4;
|
||||
}
|
||||
|
||||
return ip - ipstart;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_decompressFrame() :
|
||||
* `dctx` must be properly initialized */
|
||||
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
const void** srcPtr, size_t *srcSizePtr)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* ip = (const BYTE*)(*srcPtr);
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
size_t remainingSize = srcSize;
|
||||
size_t remainingSize = *srcSizePtr;
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
|
||||
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
|
||||
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize));
|
||||
if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
|
|
@ -1428,25 +1554,98 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
|||
if (remainingSize<4) return ERROR(checksum_wrong);
|
||||
checkRead = MEM_readLE32(ip);
|
||||
if (checkRead != checkCalc) return ERROR(checksum_wrong);
|
||||
ip += 4;
|
||||
remainingSize -= 4;
|
||||
}
|
||||
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
// Allow caller to get size read
|
||||
*srcPtr = ip;
|
||||
*srcSizePtr = remainingSize;
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void *dict, size_t dictSize,
|
||||
const ZSTD_DCtx* refContext)
|
||||
{
|
||||
void* const dststart = dst;
|
||||
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
||||
U32 magicNumber;
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) {
|
||||
size_t const frameSize = ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
|
||||
size_t decodedSize;
|
||||
if (ZSTD_isError(frameSize)) return frameSize;
|
||||
|
||||
decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
|
||||
|
||||
dst = (BYTE*)dst + decodedSize;
|
||||
dstCapacity -= decodedSize;
|
||||
|
||||
src = (const BYTE*)src + frameSize;
|
||||
srcSize -= frameSize;
|
||||
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_MAGICNUMBER) {
|
||||
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
size_t skippableSize;
|
||||
if (srcSize < ZSTD_skippableHeaderSize)
|
||||
return ERROR(srcSize_wrong);
|
||||
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
|
||||
ZSTD_skippableHeaderSize;
|
||||
if (srcSize < skippableSize) {
|
||||
return ERROR(srcSize_wrong);
|
||||
}
|
||||
|
||||
src = (const BYTE *)src + skippableSize;
|
||||
srcSize -= skippableSize;
|
||||
continue;
|
||||
} else {
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
}
|
||||
|
||||
if (refContext) {
|
||||
/* we were called from ZSTD_decompress_usingDDict */
|
||||
ZSTD_refDCtx(dctx, refContext);
|
||||
} else {
|
||||
/* this will initialize correctly with no dict if dict == NULL, so
|
||||
* use this in all cases but ddict */
|
||||
CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
|
||||
}
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
|
||||
{
|
||||
const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
|
||||
&src, &srcSize);
|
||||
if (ZSTD_isError(res)) return res;
|
||||
/* don't need to bounds check this, ZSTD_decompressFrame will have
|
||||
* already */
|
||||
dst = (BYTE*)dst + res;
|
||||
dstCapacity -= res;
|
||||
}
|
||||
}
|
||||
|
||||
if (srcSize) {
|
||||
return ERROR(srcSize_wrong);
|
||||
}
|
||||
|
||||
return (BYTE*)dst - (BYTE*)dststart;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
|
||||
#endif
|
||||
CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1843,12 +2042,10 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
|||
const void* src, size_t srcSize,
|
||||
const ZSTD_DDict* ddict)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
|
||||
#endif
|
||||
ZSTD_refDCtx(dctx, ddict->refContext);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
/* pass content and size in case legacy frames are encountered */
|
||||
return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
|
||||
ddict->dictContent, ddict->dictSize,
|
||||
ddict->refContext);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ EXPORTS
|
|||
ZSTD_getBlockSizeMax
|
||||
ZSTD_getCParams
|
||||
ZSTD_getDecompressedSize
|
||||
ZSTD_findDecompressedSize
|
||||
ZSTD_getFrameContentSize
|
||||
ZSTD_getErrorName
|
||||
ZSTD_getFrameParams
|
||||
ZSTD_getParams
|
||||
|
|
|
|||
|
|
@ -123,6 +123,30 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
|||
}
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_getFrameCompressedSizeLegacy(const void *src,
|
||||
size_t compressedSize)
|
||||
{
|
||||
U32 const version = ZSTD_isLegacy(src, compressedSize);
|
||||
switch(version)
|
||||
{
|
||||
case 1 :
|
||||
return ZSTDv01_getFrameCompressedSize(src, compressedSize);
|
||||
case 2 :
|
||||
return ZSTDv02_getFrameCompressedSize(src, compressedSize);
|
||||
case 3 :
|
||||
return ZSTDv03_getFrameCompressedSize(src, compressedSize);
|
||||
case 4 :
|
||||
return ZSTDv04_getFrameCompressedSize(src, compressedSize);
|
||||
case 5 :
|
||||
return ZSTDv05_getFrameCompressedSize(src, compressedSize);
|
||||
case 6 :
|
||||
return ZSTDv06_getFrameCompressedSize(src, compressedSize);
|
||||
case 7 :
|
||||
return ZSTDv07_getFrameCompressedSize(src, compressedSize);
|
||||
default :
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1992,6 +1992,37 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|||
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
U32 magicNumber;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
magicNumber = ZSTD_readBE32(src);
|
||||
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
||||
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTDv01_isError(blockSize)) return blockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (blockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (blockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += blockSize;
|
||||
remainingSize -= blockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/*******************************
|
||||
* Streaming Decompression API
|
||||
|
|
|
|||
|
|
@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
|||
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
||||
*/
|
||||
size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -3378,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
U32 magicNumber;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
||||
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/*******************************
|
||||
* Streaming Decompression API
|
||||
|
|
@ -3492,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
|||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv02_getFrameCompressedSize(const void *src, size_t compressedSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, compressedSize);
|
||||
}
|
||||
|
||||
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
|
||||
{
|
||||
return (ZSTDv02_Dctx*)ZSTD_createDCtx();
|
||||
|
|
|
|||
|
|
@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
|
|||
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
|
||||
*/
|
||||
size_t ZSTDv02_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -3019,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
|
|||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
U32 magicNumber;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
||||
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
|
||||
/*******************************
|
||||
* Streaming Decompression API
|
||||
|
|
@ -3133,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|||
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
}
|
||||
|
||||
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
|
||||
{
|
||||
return (ZSTDv03_Dctx*)ZSTD_createDCtx();
|
||||
|
|
|
|||
|
|
@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
|
|||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
|
||||
*/
|
||||
size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
|
||||
*/
|
||||
unsigned ZSTDv03_isError(size_t code);
|
||||
|
|
|
|||
|
|
@ -3326,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
|
|||
return op-ostart;
|
||||
}
|
||||
|
||||
static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/* ******************************
|
||||
* Streaming Decompression API
|
||||
|
|
@ -3753,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameCompressedSize(src, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
|
|||
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
|
||||
*/
|
||||
size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -3583,6 +3583,35 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
|||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv05_getFrameCompressedSize(const void *src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTDv05_blockHeaderSize;
|
||||
remainingSize -= ZSTDv05_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/* ******************************
|
||||
* Streaming Decompression API
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ extern "C" {
|
|||
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv05_isError())
|
||||
*/
|
||||
size_t ZSTDv05_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/* *************************************
|
||||
* Helper functions
|
||||
|
|
|
|||
|
|
@ -3729,6 +3729,37 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties = { bt_compressed, 0 };
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
|
||||
if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTDv06_blockHeaderSize;
|
||||
remainingSize -= ZSTDv06_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/*_******************************
|
||||
* Streaming Decompression API
|
||||
|
|
|
|||
|
|
@ -41,6 +41,13 @@ extern "C" {
|
|||
ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv06_isError())
|
||||
*/
|
||||
size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/* *************************************
|
||||
* Helper functions
|
||||
|
|
|
|||
|
|
@ -3968,6 +3968,41 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
|||
#endif
|
||||
}
|
||||
|
||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
|
||||
if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
blockProperties_t blockProperties;
|
||||
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTDv07_blockHeaderSize;
|
||||
remainingSize -= ZSTDv07_blockHeaderSize;
|
||||
|
||||
if (blockProperties.blockType == bt_end) break;
|
||||
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return ip - (const BYTE*)src;
|
||||
}
|
||||
|
||||
/*_******************************
|
||||
* Streaming Decompression API
|
||||
|
|
|
|||
|
|
@ -48,6 +48,14 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);
|
|||
ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
|
||||
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
|
||||
return : the number of bytes that would be read to decompress this frame
|
||||
or an errorCode if it fails (which can be tested using ZSTDv07_isError())
|
||||
*/
|
||||
size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t compressedSize);
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
|
||||
|
|
|
|||
76
lib/zstd.h
76
lib/zstd.h
|
|
@ -56,7 +56,7 @@ extern "C" {
|
|||
/*------ Version ------*/
|
||||
#define ZSTD_VERSION_MAJOR 1
|
||||
#define ZSTD_VERSION_MINOR 1
|
||||
#define ZSTD_VERSION_RELEASE 3
|
||||
#define ZSTD_VERSION_RELEASE 4
|
||||
|
||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||
#define ZSTD_QUOTE(str) #str
|
||||
|
|
@ -80,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
|
|||
int compressionLevel);
|
||||
|
||||
/*! ZSTD_decompress() :
|
||||
`compressedSize` : must be the _exact_ size of a single compressed frame.
|
||||
`compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
|
||||
`dstCapacity` is an upper bound of originalSize.
|
||||
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
|
||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||
|
|
@ -89,6 +89,15 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
|
|||
const void* src, size_t compressedSize);
|
||||
|
||||
/*! ZSTD_getDecompressedSize() :
|
||||
* NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
|
||||
* ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
|
||||
* frame, but distinguishes empty frames from frames with an unknown size, or errors.
|
||||
*
|
||||
* Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
|
||||
* concatenated frames in one buffer, and so is more general.
|
||||
* As a result however, it requires more computation and entire frames to be passed to it,
|
||||
* as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
|
||||
*
|
||||
* 'src' is the start of a zstd compressed frame.
|
||||
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
|
||||
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
|
|
@ -331,6 +340,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
|||
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
|
||||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
|
||||
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX_32 25
|
||||
#define ZSTD_WINDOWLOG_MAX_64 27
|
||||
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
||||
|
|
@ -384,6 +396,54 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
|||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
|
||||
/***************************************
|
||||
* Compressed size functions
|
||||
***************************************/
|
||||
|
||||
/*! ZSTD_getFrameCompressedSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame
|
||||
* `srcSize` must be at least as large as the frame
|
||||
* @return : the compressed size of the frame pointed to by `src`, suitable to pass to
|
||||
* `ZSTD_decompress` or similar, or an error code if given invalid input. */
|
||||
ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
/***************************************
|
||||
* Decompressed size functions
|
||||
***************************************/
|
||||
/*! ZSTD_getFrameContentSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame
|
||||
* `srcSize` must be at least as large as the frame header. A value greater than or equal
|
||||
* to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
|
||||
* @return : decompressed size of the frame pointed to be `src` if known, otherwise
|
||||
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
|
||||
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
|
||||
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
|
||||
/*! ZSTD_findDecompressedSize() :
|
||||
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
|
||||
* `srcSize` must be the _exact_ size of this series
|
||||
* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
|
||||
* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
|
||||
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
|
||||
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
|
||||
*
|
||||
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
|
||||
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
|
||||
* (For example, data may be necessarily cut into blocks <= 16 KB).
|
||||
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
|
||||
* note 3 : decompressed size can be very large (64-bits value),
|
||||
* potentially larger than what local system can handle as a single memory segment.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
|
||||
* Always ensure result fits within application's authorized limits.
|
||||
* Each application can set its own limits.
|
||||
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
|
||||
* read each contained frame header. This is efficient as most of the data is skipped,
|
||||
* however it does mean that all frame data must be present and valid. */
|
||||
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/***************************************
|
||||
* Advanced compression functions
|
||||
|
|
@ -517,12 +577,12 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
|||
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
|
||||
|
||||
|
||||
|
|
@ -578,9 +638,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
|
|||
/*===== Buffer-less streaming compression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||
U64 dSize64 = 0;
|
||||
U32 fileNb;
|
||||
for (fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
U64 const fSize64 = ZSTD_getDecompressedSize(srcPtr, fileSizes[fileNb]);
|
||||
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
|
||||
if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
|
||||
dSize64 += fSize64;
|
||||
srcPtr += fileSizes[fileNb];
|
||||
|
|
@ -217,7 +217,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||
blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
|
||||
blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
|
||||
blockTable[nbBlocks].resPtr = (void*)resPtr;
|
||||
blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_getDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
|
||||
blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += blockTable[nbBlocks].cRoom;
|
||||
resPtr += thisBlockSize;
|
||||
|
|
|
|||
|
|
@ -91,6 +91,32 @@ static clock_t g_time = 0;
|
|||
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
#if defined(_MSC_VER) && _MSC_VER >= 1400
|
||||
# define LONG_SEEK _fseeki64
|
||||
#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__)
|
||||
# define LONG_SEEK fseeko64
|
||||
#elif defined(_WIN32) && !defined(__DJGPP__)
|
||||
# include <windows.h>
|
||||
static int LONG_SEEK(FILE* file, __int64 offset, int origin) {
|
||||
LARGE_INTEGER off;
|
||||
DWORD method;
|
||||
off.QuadPart = offset;
|
||||
if (origin == SEEK_END)
|
||||
method = FILE_END;
|
||||
else if (origin == SEEK_CUR)
|
||||
method = FILE_CURRENT;
|
||||
else
|
||||
method = FILE_BEGIN;
|
||||
|
||||
if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method))
|
||||
return 0;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
#else
|
||||
# define LONG_SEEK fseek
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Local Parameters - Not thread safe
|
||||
|
|
@ -271,8 +297,8 @@ typedef struct {
|
|||
} cRess_t;
|
||||
|
||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
U64 srcSize, ZSTD_compressionParameters* comprParams)
|
||||
{
|
||||
U64 srcSize, int srcRegFile,
|
||||
ZSTD_compressionParameters* comprParams) {
|
||||
cRess_t ress;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
|
|
@ -298,7 +324,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
|||
size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
|
||||
if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
params.fParams.contentSizeFlag = srcRegFile;
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
|
|
@ -473,8 +499,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
|||
{
|
||||
clock_t const start = clock();
|
||||
U64 const srcSize = UTIL_getFileSize(srcFileName);
|
||||
int const regFile = UTIL_isRegFile(srcFileName);
|
||||
|
||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
|
||||
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
|
||||
|
||||
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
||||
|
|
@ -495,7 +522,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
|||
char* dstFileName = (char*)malloc(FNSPACE);
|
||||
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
||||
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||
int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
|
||||
|
||||
/* init */
|
||||
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
|
||||
|
|
@ -598,7 +626,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
|||
|
||||
/* avoid int overflow */
|
||||
if (storedSkips > 1 GB) {
|
||||
int const seekResult = fseek(file, 1 GB, SEEK_CUR);
|
||||
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
|
||||
storedSkips -= 1 GB;
|
||||
}
|
||||
|
|
@ -614,7 +642,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
|||
storedSkips += (unsigned)(nb0T * sizeof(size_t));
|
||||
|
||||
if (nb0T != seg0SizeT) { /* not all 0s */
|
||||
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
seg0SizeT -= nb0T;
|
||||
|
|
@ -634,7 +662,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
|||
for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
|
||||
storedSkips += (unsigned) (restPtr - restStart);
|
||||
if (restPtr != restEnd) {
|
||||
int seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
{ size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
|
||||
|
|
@ -647,7 +675,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
|||
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
{
|
||||
if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
|
||||
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
|
||||
{ const char lastZeroByte[1] = { 0 };
|
||||
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
|
||||
|
|
|
|||
|
|
@ -182,6 +182,11 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
|||
return 1;
|
||||
}
|
||||
|
||||
UTIL_STATIC int UTIL_isRegFile(const char* infilename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
|
||||
}
|
||||
|
||||
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ paramgrill32
|
|||
roundTripCrash
|
||||
longmatch
|
||||
symbols
|
||||
legacy
|
||||
pool
|
||||
invalidDictionaries
|
||||
|
||||
|
|
|
|||
|
|
@ -149,6 +149,11 @@ longmatch : $(ZSTD_FILES) longmatch.c
|
|||
invalidDictionaries : $(ZSTD_FILES) invalidDictionaries.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=1
|
||||
legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
|
||||
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
symbols : symbols.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
|
|
@ -225,7 +230,7 @@ zstd-playTests: datagen
|
|||
file $(ZSTD)
|
||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
|
||||
ifneq ($(QEMU_SYS),qemu-ppc64-static)
|
||||
test: test-pool
|
||||
endif
|
||||
|
|
@ -294,6 +299,9 @@ test-invalidDictionaries: invalidDictionaries
|
|||
test-symbols: symbols
|
||||
$(QEMU_SYS) ./symbols
|
||||
|
||||
test-legacy: legacy
|
||||
$(QEMU_SYS) ./legacy
|
||||
|
||||
test-pool: pool
|
||||
$(QEMU_SYS) ./pool
|
||||
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
|
||||
{ unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
|
||||
{ unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
|
||||
if (rSize != CNBuffSize) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
|
@ -167,6 +167,46 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Simple API multiframe test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++);
|
||||
{ size_t off = 0;
|
||||
int i;
|
||||
int const segs = 4;
|
||||
/* only use the first half so we don't push against size limit of compressedBuffer */
|
||||
size_t const segSize = (CNBuffSize / 2) / segs;
|
||||
for (i = 0; i < segs; i++) {
|
||||
CHECK_V(r,
|
||||
ZSTD_compress(
|
||||
(BYTE *)compressedBuffer + off, CNBuffSize - off,
|
||||
(BYTE *)CNBuffer + segSize * i,
|
||||
segSize, 5));
|
||||
off += r;
|
||||
if (i == segs/2) {
|
||||
/* insert skippable frame */
|
||||
const U32 skipLen = 128 KB;
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
|
||||
off += skipLen + ZSTD_skippableHeaderSize;
|
||||
}
|
||||
}
|
||||
cSize = off;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : get decompressed size of multiple frames : ", testNb++);
|
||||
{ unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
|
||||
if (r != CNBuffSize / 2) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress multiple frames : ", testNb++);
|
||||
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
|
||||
if (r != CNBuffSize / 2) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Dictionary and CCtx Duplication tests */
|
||||
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
|
||||
|
|
@ -180,7 +220,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||
|
||||
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) );
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
|
||||
|
|
@ -658,7 +698,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||
} }
|
||||
|
||||
/* Decompressed size test */
|
||||
{ unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize);
|
||||
{ unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
|
||||
CHECK(rSize != sampleSize, "decompressed size incorrect");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* Copyright (c) 2017-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
/*
|
||||
This program uses hard-coded data compressed with Zstd legacy versions
|
||||
and tests that the API decompresses them correctly
|
||||
*/
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <string.h> /* strlen */
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
/*===========================================
|
||||
* Macros
|
||||
*==========================================*/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
/*===========================================
|
||||
* Precompressed frames
|
||||
*==========================================*/
|
||||
const char* const COMPRESSED; /* content is at end of file */
|
||||
size_t const COMPRESSED_SIZE = 917;
|
||||
const char* const EXPECTED; /* content is at end of file */
|
||||
|
||||
|
||||
int testSimpleAPI(void)
|
||||
{
|
||||
size_t const size = strlen(EXPECTED);
|
||||
char* const output = malloc(size);
|
||||
|
||||
if (!output) {
|
||||
DISPLAY("ERROR: Not enough memory!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
{
|
||||
size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE);
|
||||
if (ZSTD_isError(ret)) {
|
||||
if (ret == ZSTD_error_prefix_unknown) {
|
||||
DISPLAY("ERROR: Invalid frame magic number, was this compiled "
|
||||
"without legacy support?\n");
|
||||
} else {
|
||||
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
if (ret != size) {
|
||||
DISPLAY("ERROR: Wrong decoded size\n");
|
||||
}
|
||||
}
|
||||
if (memcmp(EXPECTED, output, size) != 0) {
|
||||
DISPLAY("ERROR: Wrong decoded output produced\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
DISPLAY("Simple API OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int testStreamingAPI(void)
|
||||
{
|
||||
size_t const outBuffSize = ZSTD_DStreamOutSize();
|
||||
char* const outBuff = malloc(outBuffSize);
|
||||
ZSTD_DStream* const stream = ZSTD_createDStream();
|
||||
ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 };
|
||||
size_t outputPos = 0;
|
||||
int needsInit = 1;
|
||||
|
||||
if (outBuff == NULL) {
|
||||
DISPLAY("ERROR: Could not allocate memory\n");
|
||||
return 1;
|
||||
}
|
||||
if (stream == NULL) {
|
||||
DISPLAY("ERROR: Could not create dstream\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
ZSTD_outBuffer output = {outBuff, outBuffSize, 0};
|
||||
if (needsInit) {
|
||||
size_t const ret = ZSTD_initDStream(stream);
|
||||
if (ZSTD_isError(ret)) {
|
||||
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
{
|
||||
size_t const ret = ZSTD_decompressStream(stream, &output, &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (ret == 0) {
|
||||
needsInit = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) {
|
||||
DISPLAY("ERROR: Wrong decoded output produced\n");
|
||||
return 1;
|
||||
}
|
||||
outputPos += output.pos;
|
||||
if (input.pos == input.size && output.pos < output.size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAY("Streaming API OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = testSimpleAPI();
|
||||
if (ret) return ret;
|
||||
ret = testStreamingAPI();
|
||||
if (ret) return ret;
|
||||
|
||||
DISPLAY("OK\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Consists of the "EXPECTED" string compressed with default settings on
|
||||
- v0.4.3
|
||||
- v0.5.0
|
||||
- v0.6.0
|
||||
- v0.7.0
|
||||
- v0.8.0
|
||||
*/
|
||||
const char* const COMPRESSED =
|
||||
"\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01"
|
||||
"\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53"
|
||||
"\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00"
|
||||
"\x18\x00\x3F\xE6\xE2\xE3\x74\xD6\xEC\xC9\x4A\xE0\x71\x71\x42\x3E"
|
||||
"\x64\x4F\x6A\x45\x4E\x78\xEC\x49\x03\x3F\xC6\x80\xAB\x8F\x75\x5E"
|
||||
"\x6F\x2E\x3E\x7E\xC6\xDC\x45\x69\x6C\xC5\xFD\xC7\x40\xB8\x84\x8A"
|
||||
"\x01\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B\x67"
|
||||
"\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\x19\x03\x01\x50\x67\x56\xF5\x9F"
|
||||
"\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF"
|
||||
"\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24"
|
||||
"\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00"
|
||||
"\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C"
|
||||
"\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0"
|
||||
"\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF"
|
||||
"\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89"
|
||||
"\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC"
|
||||
"\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C"
|
||||
"\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A"
|
||||
"\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6"
|
||||
"\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC"
|
||||
"\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00"
|
||||
"\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87"
|
||||
"\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01"
|
||||
"\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5"
|
||||
"\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00"
|
||||
"\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9"
|
||||
"\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F"
|
||||
"\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF"
|
||||
"\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B"
|
||||
"\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7"
|
||||
"\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8"
|
||||
"\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6"
|
||||
"\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C"
|
||||
"\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B"
|
||||
"\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00"
|
||||
"\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10"
|
||||
"\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF"
|
||||
"\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32"
|
||||
"\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45"
|
||||
"\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14"
|
||||
"\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51"
|
||||
"\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB"
|
||||
"\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78"
|
||||
"\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4"
|
||||
"\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1"
|
||||
"\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0"
|
||||
"\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0"
|
||||
"\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF"
|
||||
"\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21"
|
||||
"\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20"
|
||||
"\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB"
|
||||
"\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB"
|
||||
"\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F"
|
||||
"\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E"
|
||||
"\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55"
|
||||
"\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39"
|
||||
"\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D"
|
||||
"\x01\xD2\x2F\x21\x80";
|
||||
|
||||
const char* const EXPECTED =
|
||||
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
|
||||
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
|
||||
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
|
||||
|
||||
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
|
||||
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
|
||||
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
|
||||
|
||||
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
|
||||
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
|
||||
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
|
||||
|
||||
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
|
||||
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
|
||||
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"
|
||||
|
||||
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
|
||||
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
|
||||
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n";
|
||||
|
||||
|
|
@ -15,6 +15,8 @@ static const void *symbols[] = {
|
|||
&ZSTD_compress,
|
||||
&ZSTD_decompress,
|
||||
&ZSTD_getDecompressedSize,
|
||||
&ZSTD_findDecompressedSize,
|
||||
&ZSTD_getFrameContentSize,
|
||||
&ZSTD_maxCLevel,
|
||||
&ZSTD_compressBound,
|
||||
&ZSTD_isError,
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
{ unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
|
|
@ -438,6 +438,34 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
|
||||
|
||||
/* Unknown srcSize */
|
||||
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
|
||||
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
|
||||
|
||||
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
_end:
|
||||
FUZ_freeDictionary(dictionary);
|
||||
|
|
|
|||
Loading…
Reference in New Issue