facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new field : ZSTD_frameHeader.frameType 

Makes frame type (zstd,skippable) detection more straighforward.
ZSTD_getFrameHeader set frameContentSize=ZSTD_CONTENTSIZE_UNKNOWN to mean "field not present"
dev
Yann Collet 2017-07-07 15:21:35 -07:00
parent e622330a3b
commit 990449b89d
4 changed files with 57 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/zstd_manual.html
View File

@ -344,9 +344,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
} ZSTD_parameters; } ZSTD_parameters;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
</b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned long long frameContentSize; unsigned long long frameContentSize; </b>/* ZSTD_CONTENTSIZE_UNKNOWN means this field is not available. 0 means "empty" */<b>
unsigned long long windowSize; </b>/* can be == frameContentSize */<b> unsigned long long windowSize; </b>/* can be very large, up to <= frameContentSize */<b>
ZSTD_frameType_e frameType; </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
unsigned dictID; unsigned dictID;
unsigned checksumFlag; unsigned checksumFlag;
} ZSTD_frameHeader; } ZSTD_frameHeader;

72
lib/decompress/zstd_decompress.c
View File

@ -304,7 +304,8 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
return ZSTD_skippableHeaderSize; /* magic number + frame length */ return ZSTD_skippableHeaderSize; /* magic number + frame length */
memset(zfhPtr, 0, sizeof(*zfhPtr)); memset(zfhPtr, 0, sizeof(*zfhPtr));
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4); zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
zfhPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ zfhPtr->frameType = ZSTD_skippableFrame;
zfhPtr->windowSize = 0;
return 0; return 0;
} }
return ERROR(prefix_unknown); return ERROR(prefix_unknown);
@ -321,11 +322,12 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
U32 const singleSegment = (fhdByte>>5)&1; U32 const singleSegment = (fhdByte>>5)&1;
U32 const fcsID = fhdByte>>6; U32 const fcsID = fhdByte>>6;
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
U32 windowSize = 0; U64 windowSize = 0;
U32 dictID = 0; U32 dictID = 0;
U64 frameContentSize = 0; U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
if ((fhdByte & 0x08) != 0) if ((fhdByte & 0x08) != 0)
return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */ return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */
if (!singleSegment) { if (!singleSegment) {
BYTE const wlByte = ip[pos++]; BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
@ -334,10 +336,9 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
windowSize = (1U << windowLog); windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7); windowSize += (windowSize >> 3) * (wlByte&7);
} }
switch(dictIDSizeCode) switch(dictIDSizeCode)
{ {
default: /* impossible */ default: assert(0); /* impossible */
case 0 : break; case 0 : break;
case 1 : dictID = ip[pos]; pos++; break; case 1 : dictID = ip[pos]; pos++; break;
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
@ -345,27 +346,30 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
} }
switch(fcsID) switch(fcsID)
{ {
default: /* impossible */ default: assert(0); /* impossible */
case 0 : if (singleSegment) frameContentSize = ip[pos]; break; case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
case 2 : frameContentSize = MEM_readLE32(ip+pos); break; case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
case 3 : frameContentSize = MEM_readLE64(ip+pos); break; case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
} }
if (!windowSize) windowSize = (U32)frameContentSize; if (singleSegment) windowSize = frameContentSize;
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
zfhPtr->frameType = ZSTD_frame;
zfhPtr->frameContentSize = frameContentSize; zfhPtr->frameContentSize = frameContentSize;
zfhPtr->windowSize = windowSize; zfhPtr->windowSize = windowSize;
zfhPtr->dictID = dictID; zfhPtr->dictID = dictID;
zfhPtr->checksumFlag = checksumFlag; zfhPtr->checksumFlag = checksumFlag;
if (windowSize > windowSizeMax)
return ERROR(frameParameter_windowTooLarge); /* should windowSizeMax control be delegated to caller ? */
} }
return 0; return 0;
} }
/** ZSTD_getFrameContentSize() : /** ZSTD_getFrameContentSize() :
* compatible with legacy mode * compatible with legacy mode
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
@ -374,18 +378,14 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
} }
#endif #endif
{ ZSTD_frameHeader fParams; { ZSTD_frameHeader zfh;
if (ZSTD_getFrameHeader(&fParams, src, srcSize) != 0) if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
return ZSTD_CONTENTSIZE_ERROR; return ZSTD_CONTENTSIZE_ERROR;
if (fParams.windowSize == 0) { if (zfh.frameType == ZSTD_skippableFrame) {
/* Either skippable or empty frame, size == 0 either way */
return 0; return 0;
} else if (fParams.frameContentSize != 0) {
return fParams.frameContentSize;
} else { } else {
return ZSTD_CONTENTSIZE_UNKNOWN; return zfh.frameContentSize;
} } }
}
} }
/** ZSTD_findDecompressedSize() : /** ZSTD_findDecompressedSize() :
@ -462,7 +462,8 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize); size_t const result = ZSTD_getFrameHeader(&(dctx->fParams), src, headerSize);
if (ZSTD_isError(result)) return result; /* invalid header */ if (ZSTD_isError(result)) return result; /* invalid header */
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
return 0; return 0;
} }
@ -1445,13 +1446,13 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
const BYTE* const ipstart = ip; const BYTE* const ipstart = ip;
size_t remainingSize = srcSize; size_t remainingSize = srcSize;
ZSTD_frameHeader fParams; ZSTD_frameHeader zfh;
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); size_t const headerSize = ZSTD_frameHeaderSize(src, srcSize);
if (ZSTD_isError(headerSize)) return headerSize; if (ZSTD_isError(headerSize)) return headerSize;
/* Frame Header */ /* Frame Header */
{ size_t const ret = ZSTD_getFrameHeader(&fParams, ip, remainingSize); { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(ret)) return ret; if (ZSTD_isError(ret)) return ret;
if (ret > 0) return ERROR(srcSize_wrong); if (ret > 0) return ERROR(srcSize_wrong);
} }
@ -1474,7 +1475,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
if (blockProperties.lastBlock) break; if (blockProperties.lastBlock) break;
} }
if (fParams.checksumFlag) { /* Frame content checksum */ if (zfh.checksumFlag) { /* Final frame content checksum */
if (remainingSize < 4) return ERROR(srcSize_wrong); if (remainingSize < 4) return ERROR(srcSize_wrong);
ip += 4; ip += 4;
remainingSize -= 4; remainingSize -= 4;
@ -2135,7 +2136,8 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
* ZSTD_getFrameHeader(), which will provide a more precise error code. */ * ZSTD_getFrameHeader(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{ {
ZSTD_frameHeader zfp = { 0 , 0 , 0 , 0 }; ZSTD_frameHeader zfp;
zfp.dictID = 0;
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0; if (ZSTD_isError(hError)) return 0;
return zfp.dictID; return zfp.dictID;
@ -2252,11 +2254,11 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
{ {
ZSTD_frameHeader fh; ZSTD_frameHeader zfh;
size_t const err = ZSTD_getFrameHeader(&fh, src, srcSize); size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(err)) return err; if (ZSTD_isError(err)) return err;
if (err>0) return ERROR(srcSize_wrong); if (err>0) return ERROR(srcSize_wrong);
return ZSTD_estimateDStreamSize(fh.windowSize); return ZSTD_estimateDStreamSize(zfh.windowSize);
} }
@ -2307,16 +2309,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
/* legacy support is incompatible with static dctx */ /* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation); if (zds->staticSize) return ERROR(memory_allocation);
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
zds->previousLegacyVersion, legacyVersion,
dict, dictSize)); dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
} else {
return hSize; /* error */
} }
#else
return hSize;
#endif #endif
return hSize; /* error */
} }
if (hSize != 0) { /* need more input */ if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */

7
lib/zstd.h
View File

@ -425,9 +425,12 @@ typedef struct {
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
} ZSTD_parameters; } ZSTD_parameters;
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef struct { typedef struct {
unsigned long long frameContentSize; unsigned long long frameContentSize; /* ZSTD_CONTENTSIZE_UNKNOWN means this field is not available. 0 means "empty" */
unsigned long long windowSize; /* can be == frameContentSize */ unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned dictID; unsigned dictID;
unsigned checksumFlag; unsigned checksumFlag;
} ZSTD_frameHeader; } ZSTD_frameHeader;

18
tests/fuzzer.c
View File

@ -434,9 +434,9 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize), (const char*)CNBuffer + dictSize, testSize),
cSize = r); cSize = r);
{ ZSTD_frameHeader fp; { ZSTD_frameHeader zfh;
if (ZSTD_getFrameHeader(&fp, compressedBuffer, cSize)) goto _output_error; if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error; if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
} } } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@ -1006,17 +1006,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
} } } }
/* frame header decompression test */
{ ZSTD_frameHeader zfh;
CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
}
/* Decompressed size test */ /* Decompressed size test */
{ unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
CHECK(rSize != sampleSize, "decompressed size incorrect"); CHECK(rSize != sampleSize, "decompressed size incorrect");
} }
/* frame header decompression test */
{ ZSTD_frameHeader dParams;
CHECK_Z( ZSTD_getFrameHeader(&dParams, cBuffer, cSize) );
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
}
/* successful decompression test */ /* successful decompression test */
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);