facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support for skippable frames

dev
inikep 2016-05-31 12:43:46 +02:00
parent d57b418214
commit f772bf54a5
6 changed files with 66 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd_internal.h
View File

@ -252,6 +252,6 @@ typedef struct {
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
#endif /* ZSTD_CCOMMON_H_MODULE */

4
lib/common/zstd_static.h
View File

@ -51,7 +51,8 @@ extern "C" {
/*-*************************************
* Constants
***************************************/
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
/*-*************************************
@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_min = 5;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);

24
lib/decompress/zbuff_decompress.c
View File

@ -218,12 +218,19 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize) break; /* this was just a header */
if (!decodedSize) {
if (isSkipFrame) {
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = 0;
}
break; /* this was just a header */
}
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
break;
@ -243,12 +250,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
{ const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zbd->inPos = 0; /* input is consumed */
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
if (!decodedSize) {
if (isSkipFrame) {
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = 0;
break;
}
zbd->stage = ZBUFFds_read; /* this was just a header */
break;
}
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */

35
lib/decompress/zstd_decompress.c
View File

@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
* Context management
***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
struct ZSTD_DCtx_s
{
@ -312,7 +313,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
const BYTE* ip = (const BYTE*)src;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
fparamsPtr->frameContentSize = 0;
fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;
return 0;
}
return ERROR(prefix_unknown);
}
/* ensure there is enough `srcSize` to fully read/decode frame header */
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
@ -995,6 +1004,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
return dctx->expected;
}
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
{
return dctx->stage == ZSTDds_skipFrame;
}
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
/* Sanity check */
@ -1006,6 +1020,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
{
case ZSTDds_getFrameHeaderSize :
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
dctx->stage = ZSTDds_decodeSkippableHeader;
return 0;
}
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
@ -1063,6 +1083,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->previousDstEnd = (char*)dst + rSize;
return rSize;
}
case ZSTDds_decodeSkippableHeader:
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame;
return 0;
}
case ZSTDds_skipFrame:
{ dctx->expected = ZSTD_frameHeaderSize_min;
dctx->stage = ZSTDds_getFrameHeaderSize;
return 0;
}
default:
return ERROR(GENERIC); /* impossible */
}

2
programs/fileio.c
View File

@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
continue;
}
#endif
if (magic != ZSTD_MAGICNUMBER) {
if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
if (g_overwrite) /* -df : pass-through mode */
return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
else {

11
programs/zbufftest.c
View File

@ -146,7 +146,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
@ -162,15 +163,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
}
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
/* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error;
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
cSize = genSize;
cSize += genSize;
genSize = compressedBufferSize - cSize;
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (result != 0) goto _output_error; /* error, or some data not flushed */