support for skippable frames

lib/common/zstd_internal.h

@@ -252,6 +252,6 @@ typedef struct {
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
-
+int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
 
 #endif   /* ZSTD_CCOMMON_H_MODULE */

lib/common/zstd_static.h

@@ -52,6 +52,7 @@ extern "C" {
 *  Constants
 ***************************************/
 #define ZSTD_MAGICNUMBER            0xFD2FB526   /* v0.6 */
+#define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 
 
 /*-*************************************
@@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
 #define ZSTD_FRAMEHEADERSIZE_MAX 13    /* for static allocation */
 static const size_t ZSTD_frameHeaderSize_min = 5;
 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
+static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
 
 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);

lib/decompress/zbuff_decompress.c

@@ -218,12 +218,19 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                     break;
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
+                    const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
                     size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
-                        zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
+                        zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
                         ip, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
                     ip += neededInSize;
-                    if (!decodedSize) break;   /* this was just a header */
+                    if (!decodedSize)  {
+                        if (isSkipFrame) {
+                           zbd->stage = ZBUFFds_loadHeader;
+                           zbd->lhSize = 0;
+                        }
+                        break;   /* this was just a header */
+                    }
                     zbd->outEnd = zbd->outStart +  decodedSize;
                     zbd->stage = ZBUFFds_flush;
                     break;
@@ -243,12 +250,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                 if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
-                {   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
+                {  const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
+                   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
                         zbd->inBuff, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
                     zbd->inPos = 0;   /* input is consumed */
-                    if (!decodedSize) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
+                    if (!decodedSize) {
+                        if (isSkipFrame) {
+                           zbd->stage = ZBUFFds_loadHeader;
+                           zbd->lhSize = 0;
+                           break; 
+                        }
+                        zbd->stage = ZBUFFds_read; /* this was just a header */
+                        break; 
+                    }
                     zbd->outEnd = zbd->outStart +  decodedSize;
                     zbd->stage = ZBUFFds_flush;
                     // break; /* ZBUFFds_flush follows */

lib/decompress/zstd_decompress.c

@@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
 *   Context management
 ***************************************************************/
 typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
-               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
+               ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
+               ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
 
 struct ZSTD_DCtx_s
 {
@@ -312,7 +313,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
     const BYTE* ip = (const BYTE*)src;
 
     if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
-    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
+    if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
+        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
+            fparamsPtr->frameContentSize = 0;
+            fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;
+            return 0;
+        }
+        return ERROR(prefix_unknown);
+    }
 
     /* ensure there is enough `srcSize` to fully read/decode frame header */
     { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
@@ -995,6 +1004,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
     return dctx->expected;
 }
 
+int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
+{
+    return dctx->stage == ZSTDds_skipFrame;
+}
+
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     /* Sanity check */
@@ -1006,6 +1020,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
     {
     case ZSTDds_getFrameHeaderSize :
         if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
+        if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+            dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
+            dctx->stage = ZSTDds_decodeSkippableHeader;
+            return 0;
+        }
         dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
         if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
         memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
@@ -1063,6 +1083,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
             dctx->previousDstEnd = (char*)dst + rSize;
             return rSize;
         }
+    case ZSTDds_decodeSkippableHeader:
+        {   memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
+            dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
+            dctx->stage = ZSTDds_skipFrame;
+            return 0;
+        }
+    case ZSTDds_skipFrame:
+        {   dctx->expected = ZSTD_frameHeaderSize_min;
+            dctx->stage = ZSTDds_getFrameHeaderSize;
+            return 0;
+        }
     default:
         return ERROR(GENERIC);   /* impossible */
     }

programs/fileio.c

@@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
                 continue;
             }
 #endif
-            if (magic !=  ZSTD_MAGICNUMBER) {
+            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
                 if (g_overwrite)   /* -df : pass-through mode */
                     return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
                 else {

programs/zbufftest.c

@@ -146,7 +146,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     int testResult = 0;
     size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
-    size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
+    size_t const skippableFrameSize = 11;
+    size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
     void* compressedBuffer = malloc(compressedBufferSize);
     size_t const decodedBufferSize = CNBufferSize;
     void* decodedBuffer = malloc(decodedBufferSize);
@@ -162,15 +163,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }
     RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
 
+    /* generate skippable frame */
+    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
+    MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize);
+    cSize = skippableFrameSize + 8;
     /* Basic compression test */
     DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
     readSize = CNBufferSize;
     genSize = compressedBufferSize;
-    result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
+    result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
     if (ZBUFF_isError(result)) goto _output_error;
     if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
-    cSize = genSize;
+    cSize += genSize;
     genSize = compressedBufferSize - cSize;
     result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
     if (result != 0) goto _output_error;   /* error, or some data not flushed */