Merge remote-tracking branch 'refs/remotes/Cyan4973/dev070' into dev070

# Conflicts:
#	.gitignore
#	lib/decompress/zstd_decompress.c
#	programs/zbufftest.c

.gitignore

@@ -31,7 +31,7 @@ projects/VS2015
 .directory
 _codelite/
 _zstdbench/
-zlib_wrapper/
+.clang_complete
 
 # CMake
 projects/cmake/

lib/common/error_private.h

@@ -102,6 +102,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size incorrect";
     case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
     case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";

lib/common/error_public.h

@@ -54,6 +54,7 @@ typedef enum {
   ZSTD_error_dstSize_tooSmall,
   ZSTD_error_srcSize_wrong,
   ZSTD_error_corruption_detected,
+  ZSTD_error_checksum_wrong,
   ZSTD_error_tableLog_tooLarge,
   ZSTD_error_maxSymbolValue_tooLarge,
   ZSTD_error_maxSymbolValue_tooSmall,

lib/common/zstd_static.h

@@ -87,8 +87,9 @@ typedef struct {
 } ZSTD_compressionParameters;
 
 typedef struct {
-    U32 contentSizeFlag;   /* 1: content size will be in frame header (if known). */
+    U32 contentSizeFlag;  /* 1: content size will be in frame header (if known). */
-    U32 noDictIDFlag;      /* 1: no dict ID will be saved into frame header (if dictionary compression) */
+    U32 checksumFlag;     /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+    U32 noDictIDFlag;     /* 1: no dict ID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;
 
 typedef struct {
@@ -197,6 +198,7 @@ typedef struct {
     U64 frameContentSize;
     U32 windowLog;
     U32 dictID;
+    U32 checksumFlag;
 } ZSTD_frameParams;
 
 #define ZSTD_FRAMEHEADERSIZE_MAX 18    /* for static allocation */

lib/compress/zbuff_compress.c

@@ -125,7 +125,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
     zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx));
     if (zbc==NULL) return NULL;
     memset(zbc, 0, sizeof(ZBUFF_CCtx));
-    zbc->customAlloc = customMem.customAlloc; 
+    zbc->customAlloc = customMem.customAlloc;
     zbc->customFree = customMem.customFree;
     zbc->zc = ZSTD_createCCtx_advanced(customMem);
     return zbc;
@@ -180,6 +180,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
 size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters params;
+    memset(&params, 0, sizeof(params));
     params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
     params.fParams.contentSizeFlag = 0;
     ZSTD_adjustCParams(&params.cParams, 0, dictSize);

lib/compress/zstd_compress.c

@@ -51,9 +51,11 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
-#include <stdlib.h>   /* malloc */
+#include <stdlib.h>      /* malloc */
-#include <string.h>   /* memset */
+#include <string.h>      /* memset */
 #include "mem.h"
+#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#include "xxhash.h"      /* XXH_reset, update, digest */
 #include "fse_static.h"
 #include "huf_static.h"
 #include "zstd_internal.h"
@@ -104,6 +106,7 @@ struct ZSTD_CCtx_s
     void* workSpace;
     size_t workSpaceSize;
     size_t blockSize;
+    XXH64_state_t xxhState;
     ZSTD_allocFunction customAlloc;
     ZSTD_freeFunction customFree;
 
@@ -266,6 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     }   }
 
     if (reset) memset(zc->workSpace, 0, tableSpace );   /* reset only tables */
+    XXH64_reset(&zc->xxhState, 0);
     zc->hashTable3 = (U32*)(zc->workSpace);
     zc->hashTable = zc->hashTable3 + h3Size;
     zc->chainTable = zc->hashTable + hSize;
@@ -1938,7 +1942,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
 
         /* catch up */
         if (offset) {
-            U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
+            U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
             const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
             const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
             while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; }  /* catch up */
@@ -2043,19 +2047,22 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
 
 
 
-static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
+static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                                        void* dst, size_t dstCapacity,
+                                     void* dst, size_t dstCapacity,
-                                  const void* src, size_t srcSize)
+                               const void* src, size_t srcSize)
 {
-    size_t blockSize = zc->blockSize;
+    size_t blockSize = cctx->blockSize;
     size_t remaining = srcSize;
     const BYTE* ip = (const BYTE*)src;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
-    const U32 maxDist = 1 << zc->params.cParams.windowLog;
+    const U32 maxDist = 1 << cctx->params.cParams.windowLog;
-    ZSTD_stats_t* stats = &zc->seqStore.stats;
+    ZSTD_stats_t* stats = &cctx->seqStore.stats;
     ZSTD_statsInit(stats);
 
+    if (cctx->params.fParams.checksumFlag)
+        XXH64_update(&cctx->xxhState, src, srcSize);
+
     while (remaining) {
         size_t cSize;
         ZSTD_statsResetFreqs(stats);
@@ -2063,14 +2070,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
         if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
 
-        if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
+        if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
             /* enforce maxDist */
-            U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
+            U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
-            if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit;
+            if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
-            if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
+            if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
         }
 
-        cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
+        cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
         if (ZSTD_isError(cSize)) return cSize;
 
         if (cSize == 0) {  /* block is not compressible */
@@ -2090,7 +2097,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
         op += cSize;
     }
 
-    ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
+    ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
     return op-ostart;
 }
 
@@ -2104,7 +2111,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN)   /* windowLog : 4 KB - 128 MB */
                                   | (fcsId << 6) );
     U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536);   /* 0-3 */
-    BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3);
+    BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4));
     size_t pos;
 
     if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
@@ -2261,40 +2268,45 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
 static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
 {
     /* note : magic number already checked */
-    size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
+    size_t const dictSizeStart = dictSize;
-    short offcodeNCount[MaxOff+1];
-    unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
-    short matchlengthNCount[MaxML+1];
-    unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
-    short litlengthNCount[MaxLL+1];
-    unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
 
-    size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
+    {   size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
-    if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
+        if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
-    zc->flagStaticTables = 1;
+        zc->flagStaticTables = 1;
-    dict = (const char*)dict + hufHeaderSize;
+        dict = (const char*)dict + hufHeaderSize;
-    dictSize -= hufHeaderSize;
+        dictSize -= hufHeaderSize;
+    }
 
-    offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
+    {   short offcodeNCount[MaxOff+1];
-    if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
+        unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
-    errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
+        size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
-    if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+        if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
-    dict = (const char*)dict + offcodeHeaderSize;
+        { size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
-    dictSize -= offcodeHeaderSize;
+          if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+        dict = (const char*)dict + offcodeHeaderSize;
+        dictSize -= offcodeHeaderSize;
+    }
 
-    matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
+    {   short matchlengthNCount[MaxML+1];
-    if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
+        unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
-    errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
+        size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
-    if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+        if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
-    dict = (const char*)dict + matchlengthHeaderSize;
+        { size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
-    dictSize -= matchlengthHeaderSize;
+          if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+        dict = (const char*)dict + matchlengthHeaderSize;
+        dictSize -= matchlengthHeaderSize;
+    }
 
-    litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+    {   short litlengthNCount[MaxLL+1];
-    if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+        unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
-    errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
+        size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
-    if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
+        if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
+        { size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
+          if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+        dictSize -= litlengthHeaderSize;
+    }
 
-    return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
+    return (dictSizeStart-dictSize);
 }
 
 /** ZSTD_compress_insertDictionary() :
@@ -2366,30 +2378,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
 /*! ZSTD_compressEnd() :
 *   Write frame epilogue.
 *   @return : nb of bytes written into dst (or an error code) */
-size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
+size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
 {
     BYTE* op = (BYTE*)dst;
     size_t fhSize = 0;
 
     /* not even init ! */
-    if (zc->stage==0) return ERROR(stage_wrong);
+    if (cctx->stage==0) return ERROR(stage_wrong);
 
     /* special case : empty frame */
-    if (zc->stage==1) {
+    if (cctx->stage==1) {
-        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0);
+        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
         if (ZSTD_isError(fhSize)) return fhSize;
         dstCapacity -= fhSize;
         op += fhSize;
-        zc->stage = 2;
+        cctx->stage = 2;
     }
 
     /* frame epilogue */
     if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
-    op[0] = (BYTE)(bt_end << 6);
+    {   U32 const checksum = cctx->params.fParams.checksumFlag ?
-    op[1] = 0;
+                             (U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
-    op[2] = 0;
+                             0;
+        op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
+        op[1] = (BYTE)(checksum>>8);
+        op[2] = (BYTE)checksum;
+    }
 
-    zc->stage = 0;  /* return to "created by not init" status */
+    cctx->stage = 0;  /* return to "created but not init" status */
     return 3+fhSize;
 }
 

lib/decompress/zstd_decompress.c

@@ -57,6 +57,8 @@
 #include <string.h>      /* memcpy, memmove */
 #include <stdio.h>       /* debug only : printf */
 #include "mem.h"         /* low level memory routines */
+#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#include "xxhash.h"      /* XXH64_* */
 #include "zstd_internal.h"
 #include "fse_static.h"
 #include "huf_static.h"
@@ -117,6 +119,7 @@ struct ZSTD_DCtx_s
     size_t expected;
     size_t headerSize;
     ZSTD_frameParams fParams;
+    XXH64_state_t xxhState;
     ZSTD_allocFunction customAlloc;
     ZSTD_freeFunction customFree;
     blockType_t bType;   /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
@@ -146,12 +149,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     return 0;
 }
 
-ZSTD_DCtx* ZSTD_createDCtx(void)
-{
-    ZSTD_customMem customMem = { NULL, NULL };
-    return ZSTD_createDCtx_advanced(customMem);
-}
-
 ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 {
     ZSTD_DCtx* dctx;
@@ -178,6 +175,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
     return dctx;
 }
 
+ZSTD_DCtx* ZSTD_createDCtx(void)
+{
+    ZSTD_customMem const customMem = { NULL, NULL };
+    return ZSTD_createDCtx_advanced(customMem);
+}
+
 
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
@@ -346,8 +349,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
         BYTE const checkByte = ip[5];
         size_t pos = ZSTD_frameHeaderSize_min;
         U32 const dictIDSizeCode = checkByte&3;
-        fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
         if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported);   /* reserved bits */
+        if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported);   /* reserved bits */
+        fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+        fparamsPtr->checksumFlag = checkByte & 0x10;
         switch(dictIDSizeCode)  /* fcsId */
         {
             default:   /* impossible */
@@ -376,6 +381,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr
     size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
     if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
     if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
+    if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
     return result;
 }
 
@@ -964,6 +970,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         if (cBlockSize == 0) break;   /* bt_end */
 
         if (ZSTD_isError(decodedSize)) return decodedSize;
+        if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
         op += decodedSize;
         ip += cBlockSize;
         remainingSize -= cBlockSize;
@@ -1035,6 +1042,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
     return dctx->stage == ZSTDds_skipFrame;
 }
 
+/** ZSTD_decompressContinue() :
+*   @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+*             or an error code, which can be tested using ZSTD_isError() */
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     /* Sanity check */
@@ -1076,6 +1086,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
             size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
             if (ZSTD_isError(cBlockSize)) return cBlockSize;
             if (bp.blockType == bt_end) {
+                if (dctx->fParams.checksumFlag) {
+                    U64 const h64 = XXH64_digest(&dctx->xxhState);
+                    U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
+                    const BYTE* const ip = (const BYTE*)src;
+                    U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
+                    if (check32 != h32) return ERROR(checksum_wrong);
+                }
                 dctx->expected = 0;
                 dctx->stage = ZSTDds_getFrameHeaderSize;
             } else {
@@ -1107,6 +1124,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
             dctx->stage = ZSTDds_decodeBlockHeader;
             dctx->expected = ZSTD_blockHeaderSize;
             dctx->previousDstEnd = (char*)dst + rSize;
+            if (ZSTD_isError(rSize)) return rSize;
+            if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
             return rSize;
         }
     case ZSTDds_decodeSkippableHeader:
@@ -1134,18 +1153,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
     dctx->previousDstEnd = (const char*)dict + dictSize;
 }
 
-static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart)
 {
-    size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
+    size_t dictSize = dictSizeStart;
 
-    hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
+    {   size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
-    if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
+        if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
-    dict = (const char*)dict + hSize;
+        dict = (const char*)dict + hSize;
-    dictSize -= hSize;
+        dictSize -= hSize;
+    }
 
     {   short offcodeNCount[MaxOff+1];
         U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
-        offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
+        size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
           if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@@ -1155,7 +1175,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
 
     {   short matchlengthNCount[MaxML+1];
         unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
-        matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
+        size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
           if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@@ -1165,14 +1185,15 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
 
     {   short litlengthNCount[MaxLL+1];
         unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
-        litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
+        size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
           if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
+        dictSize -= litlengthHeaderSize;
     }
 
     dctx->flagRepeatTable = 1;
-    return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
+    return dictSizeStart - dictSize;
 }
 
 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)

programs/.clang_complete

@@ -1,3 +0,0 @@
--I../lib/common
--I../lib/legacy
--I./legacy

programs/.gitignore

@@ -40,6 +40,7 @@ grillResults.txt
 _*
 tmp*
 *.zst
+result
 
 # fuzzer
 afl

programs/fuzzer.c

@@ -216,7 +216,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
             {   ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
-                ZSTD_frameParameters const fPar = { 1 , 0 };
+                ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
                 ZSTD_parameters p;
                 p.cParams = cPar; p.fParams = fPar;
                 CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
@@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
-        {   ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ };
+        {   ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
             ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
             ZSTD_parameters p;
             p.cParams = cParams; p.fParams = fParams;
@@ -639,12 +639,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
             dictSize = FUZ_randomLength(&lseed, maxSampleLog);   /* needed also for decompression */
             dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
 
-            if (FUZ_rand(&lseed) & 15) {
+            if (FUZ_rand(&lseed) & 0xF) {
                 size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
                 CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
             } else {
                 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
-                ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 };   /* note : since dictionary is fake, dictIDflag has no impact */
+                ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
+                                                    !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
+                                                    0 /*NodictID*/ };   /* note : since dictionary is fake, dictIDflag has no impact */
                 ZSTD_parameters p;
                 size_t errorCode;
                 p.cParams = cPar; p.fParams = fpar;

programs/zbufftest.c

@@ -97,7 +97,7 @@ static U32 FUZ_GetMilliStart(void)
 
 static U32 FUZ_GetMilliSpan(U32 nTimeStart)
 {
-    U32 nCurrent = FUZ_GetMilliStart();
+    U32 const nCurrent = FUZ_GetMilliStart();
     U32 nSpan = nCurrent - nTimeStart;
     if (nTimeStart > nCurrent)
         nSpan += 0x100000 * 1000;
@@ -152,7 +152,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     void* compressedBuffer = malloc(compressedBufferSize);
     size_t const decodedBufferSize = CNBufferSize;
     void* decodedBuffer = malloc(decodedBufferSize);
-    size_t result, cSize, readSize, readSkipSize, genSize;
+    size_t cSize, readSize, readSkipSize, genSize;
     U32 testNb=0;
     ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
     ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
@@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
     readSize = CNBufferSize;
     genSize = compressedBufferSize;
-    result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
+    { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
-    if (ZBUFF_isError(result)) goto _output_error;
+      if (ZBUFF_isError(r)) goto _output_error; }
     if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
     cSize += genSize;
     genSize = compressedBufferSize - cSize;
-    result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
+    { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
-    if (result != 0) goto _output_error;   /* error, or some data not flushed */
+      if (r != 0) goto _output_error; }  /*< error, or some data not flushed */
     cSize += genSize;
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
@@ -188,13 +188,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
     readSkipSize = cSize;
     genSize = CNBufferSize;
-    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
+    { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
-    if (genSize != 0) goto _output_error;   /* skippable frame */
+      if (r != 0) goto _output_error; }
+    if (genSize != 0) goto _output_error;   /* skippable frame len is 0 */
     ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
     readSize = cSize - readSkipSize;
     genSize = CNBufferSize;
-    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
+    { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
-    if (result != 0) goto _output_error;  /* should reach end of frame == 0; otherwise, some data left, or an error */
+      if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
     if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
     if (readSize+readSkipSize != cSize) goto _output_error;   /* should have read the entire frame */
     DISPLAYLEVEL(4, "OK \n");
@@ -613,9 +614,10 @@ int main(int argc, const char** argv)
 
     if (testNb==0) {
         result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
-        if (!result)
+        if (!result) {
+            DISPLAYLEVEL(4, "Unit tests using customMem :\n")
             result = basicUnitTests(0, ((double)proba) / 100, customMem);  /* use custom memory allocation functions */
-    }
+    }   }
 
     if (!result)
         result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);