Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

NEWS

@@ -1,6 +1,7 @@
 v0.6.1
 Fixed : Legacy codec v0.5 compatible with dictionary decompression
 Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
 New : midipix support
 
 v0.6.0

lib/common/bitstream.h

@@ -84,7 +84,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 
 /* Start with initCStream, providing the size of buffer to write into.
 *  bitStream will never write outside of this buffer.
-*  `dstCapacity` must be >= sizeof(size_t), otherwise @return will be an error code.
+*  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
 *
 *  bits are first added to a local register.
 *  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
@@ -128,7 +128,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 *  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
 *  You can then retrieve bitFields stored into the local register, **in reverse order**.
 *  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
-*  A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
+*  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
 *  Otherwise, it can be less than that, so proceed accordingly.
 *  Checking if DStream has reached its end can be performed with BIT_endOfDStream().
 */
@@ -263,9 +263,9 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
 {
     if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
 
-    if (srcSize >=  sizeof(size_t)) {  /* normal case */
+    if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
         bitD->start = (const char*)srcBuffer;
-        bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(size_t);
+        bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
           if (lastByte == 0) return ERROR(GENERIC);   /* endMark not present */
@@ -276,63 +276,58 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
-            case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
+            case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
-            case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
+            case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
-            case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
+            case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
-            case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
+            case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
-            case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
+            case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
-            case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) <<  8;
+            case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
             default:;
         }
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
           if (lastByte == 0) return ERROR(GENERIC);   /* endMark not present */
           bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
-        bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
+        bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
     }
 
     return srcSize;
 }
 
-MEM_STATIC size_t BIT_getUpperBits(size_t bitD, U32 const start)
+MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 {
-    return bitD >> start;
+    return bitContainer >> start;
 }
 
-MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start)
+MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
 #if defined(__BMI__) && defined(__GNUC__)   /* experimental */
 #  if defined(__x86_64__)
-    if (sizeof(bitD)==8)
+    if (sizeof(bitContainer)==8)
-        return _bextr_u64(bitD, start, nbBits);
+        return _bextr_u64(bitContainer, start, nbBits);
     else
 #  endif
-        return _bextr_u32(bitD, start, nbBits);
+        return _bextr_u32(bitContainer, start, nbBits);
 #else
-    return (bitD >> start) & BIT_mask[nbBits];
+    return (bitContainer >> start) & BIT_mask[nbBits];
 #endif
 }
 
-MEM_STATIC size_t BIT_getLowerBits(size_t bitD, U32 const nbBits)
+MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
-    return bitD & BIT_mask[nbBits];
+    return bitContainer & BIT_mask[nbBits];
 }
 
 /*! BIT_lookBits() :
  *  Provides next n bits from local register.
- *  local register is not modified (bits are still present for next read/look).
+ *  local register is not modified.
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
  *  @return : value extracted
  */
  MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
+#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
-#  if defined(__x86_64__)
+    return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
-    if (sizeof(bitD->bitContainer)==8)
-        return _bextr_u64(bitD->bitContainer, 64 - bitD->bitsConsumed - nbBits, nbBits);
-    else
-# endif
-        return _bextr_u32(bitD->bitContainer, 32 - bitD->bitsConsumed - nbBits, nbBits);
 #else
     U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
     return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
@@ -377,7 +372,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 *   Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
 *   This function is safe, it guarantees it will not read beyond src buffer.
 *   @return : status of `BIT_DStream_t` internal register.
-              if status == unfinished, internal register is filled with >= (sizeof(size_t)*8 - 7) bits */
+              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
 	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should never happen */

lib/common/error_private.h

@@ -78,17 +78,17 @@ typedef ZSTD_ErrorCode ERR_enum;
 
 ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
 
-ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
 
 
 /*-****************************************
 *  Error Strings
 ******************************************/
 
-ERR_STATIC const char* ERR_getErrorName(size_t code)
+ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
 {
     static const char* notErrorCode = "Unspecified error code";
-    switch( ERR_getError(code) )
+    switch( code )
     {
     case PREFIX(no_error): return "No error detected";
     case PREFIX(GENERIC):  return "Error (generic)";
@@ -107,10 +107,14 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
     case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
     case PREFIX(maxCode):
-    default: return notErrorCode;   /* impossible, due to ERR_getError() */
+    default: return notErrorCode;
     }
 }
 
+ERR_STATIC const char* ERR_getErrorName(size_t code)
+{
+    return ERR_getErrorString(ERR_getErrorCode(code));
+}
 
 #if defined (__cplusplus)
 }

lib/common/fse_decompress.c

@@ -64,6 +64,7 @@
 /* **************************************************************
 *  Error Management
 ****************************************************************/
+#define FSE_isError ERR_isError
 #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
 
@@ -110,7 +111,6 @@ void FSE_freeDTable (FSE_DTable* dt)
 
 size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
 {
-    FSE_DTableHeader DTableH;
     void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
     FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
     U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
@@ -124,19 +124,21 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
 
     /* Init, lay down lowprob symbols */
-    DTableH.tableLog = (U16)tableLog;
+    {   FSE_DTableHeader DTableH;
-    DTableH.fastMode = 1;
+        DTableH.tableLog = (U16)tableLog;
-    {   S16 const largeLimit= (S16)(1 << (tableLog-1));
+        DTableH.fastMode = 1;
-        U32 s;
+        {   S16 const largeLimit= (S16)(1 << (tableLog-1));
-        for (s=0; s<maxSV1; s++) {
+            U32 s;
-            if (normalizedCounter[s]==-1) {
+            for (s=0; s<maxSV1; s++) {
-                tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
+                if (normalizedCounter[s]==-1) {
-                symbolNext[s] = 1;
+                    tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
-            } else {
+                    symbolNext[s] = 1;
-                if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
+                } else {
-                symbolNext[s] = normalizedCounter[s];
+                    if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
-    }   }   }
+                    symbolNext[s] = normalizedCounter[s];
-    memcpy(dt, &DTableH, sizeof(DTableH));
+        }   }   }
+        memcpy(dt, &DTableH, sizeof(DTableH));
+    }
 
     /* Spread symbols */
     {   U32 const tableMask = tableSize-1;

lib/common/mem.h

@@ -63,7 +63,7 @@ extern "C" {
 /*-**************************************************************
 *  Basic Types
 *****************************************************************/
-#if !defined (__VMS) && ( defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 # include <stdint.h>
   typedef  uint8_t BYTE;
   typedef uint16_t U16;
@@ -108,8 +108,8 @@ extern "C" {
 #  endif
 #endif
 
-MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
-MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
 
 MEM_STATIC unsigned MEM_isLittleEndian(void)
 {
@@ -187,6 +187,48 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
 
 #endif /* MEM_FORCE_MEMORY_ACCESS */
 
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_ulong(in);
+#elif defined (__GNUC__)
+    return __builtin_bswap32(in);
+#else
+    return  ((in << 24) & 0xff000000 ) |
+            ((in <<  8) & 0x00ff0000 ) |
+            ((in >>  8) & 0x0000ff00 ) |
+            ((in >> 24) & 0x000000ff );
+#endif
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_uint64(in);
+#elif defined (__GNUC__)
+    return __builtin_bswap64(in);
+#else
+    return  ((in << 56) & 0xff00000000000000ULL) |
+            ((in << 40) & 0x00ff000000000000ULL) |
+            ((in << 24) & 0x0000ff0000000000ULL) |
+            ((in << 8)  & 0x000000ff00000000ULL) |
+            ((in >> 8)  & 0x00000000ff000000ULL) |
+            ((in >> 24) & 0x0000000000ff0000ULL) |
+            ((in >> 40) & 0x000000000000ff00ULL) |
+            ((in >> 56) & 0x00000000000000ffULL);
+#endif
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_swap32((U32)in);
+    else
+        return (size_t)MEM_swap64((U64)in);
+}
+
+/*=== Little endian r/w ===*/
+
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
@@ -212,51 +254,32 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_read32(memPtr);
-    else {
+    else
-        const BYTE* p = (const BYTE*)memPtr;
+        return MEM_swap32(MEM_read32(memPtr));
-        return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
-    }
 }
 
 MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
 {
-    if (MEM_isLittleEndian()) {
+    if (MEM_isLittleEndian())
         MEM_write32(memPtr, val32);
-    } else {
+    else
-        BYTE* p = (BYTE*)memPtr;
+        MEM_write32(memPtr, MEM_swap32(val32));
-        p[0] = (BYTE)val32;
-        p[1] = (BYTE)(val32>>8);
-        p[2] = (BYTE)(val32>>16);
-        p[3] = (BYTE)(val32>>24);
-    }
 }
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_read64(memPtr);
-    else {
+    else
-        const BYTE* p = (const BYTE*)memPtr;
+        return MEM_swap64(MEM_read64(memPtr));
-        return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
-                     + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
-    }
 }
 
 MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
 {
-    if (MEM_isLittleEndian()) {
+    if (MEM_isLittleEndian())
         MEM_write64(memPtr, val64);
-    } else {
+    else
-        BYTE* p = (BYTE*)memPtr;
+        MEM_write64(memPtr, MEM_swap64(val64));
-        p[0] = (BYTE)val64;
-        p[1] = (BYTE)(val64>>8);
-        p[2] = (BYTE)(val64>>16);
-        p[3] = (BYTE)(val64>>24);
-        p[4] = (BYTE)(val64>>32);
-        p[5] = (BYTE)(val64>>40);
-        p[6] = (BYTE)(val64>>48);
-        p[7] = (BYTE)(val64>>56);
-    }
 }
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
@@ -275,7 +298,58 @@ MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
         MEM_writeLE64(memPtr, (U64)val);
 }
 
- /* function safe only for comparisons */
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap32(MEM_read32(memPtr));
+    else
+        return MEM_read32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+        MEM_write32(memPtr, MEM_swap32(val32));
+    else
+        MEM_write32(memPtr, val32);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap64(MEM_read64(memPtr));
+    else
+        return MEM_read64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+        MEM_write64(memPtr, MEM_swap64(val64));
+    else
+        MEM_write64(memPtr, val64);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readBE32(memPtr);
+    else
+        return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeBE32(memPtr, (U32)val);
+    else
+        MEM_writeBE64(memPtr, (U64)val);
+}
+
+
+/* function safe only for comparisons */
 MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
 {
     switch (length)

lib/common/zbuff.h

@@ -153,7 +153,7 @@ ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
 ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
 
 /** Functions below provide recommended buffer sizes for Compression or Decompression operations.
-*   These sizes are just hints, and tend to offer better latency */
+*   These sizes are just hints, they tend to offer better latency */
 ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
 ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
 ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);

lib/common/zstd_common.c

@@ -30,17 +30,23 @@
 */
 
 
-/* *************************************
+/*-*************************************
 *  Dependencies
 ***************************************/
 #include <stdlib.h>
 #include "mem.h"
 #include "fse_static.h"  /* FSE_MIN_TABLELOG */
 #include "error_private.h"
-#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */
+#include "zstd.h"  /* declaration of ZSTD_isError, ZSTD_getErrorName */
 #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
-#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */
+#include "fse.h"   /* declaration of FSE_isError, FSE_getErrorName */
-#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */
+#include "huf.h"   /* declaration of HUF_isError, HUF_getErrorName */
+
+
+/*-****************************************
+*  Version
+******************************************/
+unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
 
 
 /*-****************************************
@@ -50,13 +56,17 @@
 *   tells if a return value is an error code */
 unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
 
+/*! ZSTD_getErrorName() :
+*   provides error code string from function result (useful for debugging) */
+const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+
 /*! ZSTD_getError() :
 *   convert a `size_t` function result into a proper ZSTD_errorCode enum */
-ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); }
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 
-/*! ZSTD_getErrorName() :
+/*! ZSTD_getErrorString() :
-*   provides error code string (useful for debugging) */
+*   provides error code string from enum */
-const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
 
 
 /*-****************************************
@@ -84,7 +94,7 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error
 
 
 /*-**************************************************************
-*  FSE NCount encoding-decoding
+*  FSE NCount decoding
 ****************************************************************/
 static short FSE_abs(short a) { return a<0 ? -a : a; }
 
@@ -171,7 +181,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
                 ip = iend - 4;
             }
             bitStream = MEM_readLE32(ip) >> (bitCount & 31);
-    }   }
+    }   }   /* while ((remaining>1) && (charnum<=*maxSVPtr)) */
     if (remaining != 1) return ERROR(GENERIC);
     *maxSVPtr = charnum-1;
 

lib/common/zstd_static.h

@@ -261,7 +261,8 @@ ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCa
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare directly with enum list published into "error_public.h" */
-ZSTDLIB_API ZSTD_ErrorCode ZSTD_getError(size_t code);
+ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #if defined (__cplusplus)

lib/decompress/huf_decompress.c

@@ -1131,7 +1131,7 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
 
     {   U32 algoNb = 0;
         if (Dtime[1] < Dtime[0]) algoNb = 1;
-        if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
+        // if (Dtime[2] < Dtime[algoNb]) algoNb = 2;   /* current speed of HUF_decompress4X6 is not good */
         return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
     }
 

lib/decompress/zbuff_decompress.c

@@ -64,14 +64,14 @@
 *           just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
 * *******************************************************************************/
 
-typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
+typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
                ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
 
 /* *** Resource management *** */
 struct ZBUFF_DCtx_s {
     ZSTD_DCtx* zd;
     ZSTD_frameParams fParams;
-    size_t blockSize;
+    ZBUFF_dStage stage;
     char*  inBuff;
     size_t inBuffSize;
     size_t inPos;
@@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s {
     size_t outBuffSize;
     size_t outStart;
     size_t outEnd;
-    ZBUFF_dStage stage;
+    size_t blockSize;
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+    size_t lhSize;
 };   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
 
 
@@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
 
 size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
 {
-    zbd->stage = ZBUFFds_readHeader;
+    zbd->stage = ZBUFFds_loadHeader;
-    zbd->inPos = zbd->outStart = zbd->outEnd = 0;
+    zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
     return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
 }
 
@@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
         case ZBUFFds_init :
             return ERROR(init_missing);
 
-        case ZBUFFds_readHeader :
+        case ZBUFFds_loadHeader :
-            /* read header from src */
+            {   size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
-            {   size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr);
+                if (hSize != 0) {
-                if (ZSTD_isError(headerSize)) return headerSize;
+                    size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
-                if (headerSize) {
+                    if (ZSTD_isError(hSize)) return hSize;
-                    /* not enough input to decode header : needs headerSize > *srcSizePtr */
+                    if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
-                    *dstCapacityPtr = 0;
+                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
-                    *srcSizePtr = 0;
+                        zbd->lhSize += iend-ip; ip = iend; notDone = 0;
-                    return headerSize;
+                        *dstCapacityPtr = 0;
+                        return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
+                    }
+                    memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
+                    break;
+            }   }
+
+            /* Consume header */
+            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);  /* == ZSTD_frameHeaderSize_min */
+                size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
+                if (ZSTD_isError(h1Result)) return h1Result;
+                if (h1Size < zbd->lhSize) {   /* long header */
+                    size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
+                    size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
+                    if (ZSTD_isError(h2Result)) return h2Result;
             }   }
 
             /* Frame header instruct buffer sizes */
@@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                     notDone = 0;
                     break;
                 }
-                if ((size_t)(iend-ip) >= neededInSize) {
+                if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
-                    /* directly decode from src */
                     size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
                         ip, neededInSize);
@@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                 ip += loadedSize;
                 zbd->inPos += loadedSize;
                 if (loadedSize < toLoad) { notDone = 0; break; }   /* not enough input, wait for more */
+
                 /* decode loaded input */
                 {   size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,

lib/decompress/zstd_decompress.c

@@ -83,18 +83,20 @@
 #endif
 
 
+/*-*************************************
+*  Macros
+***************************************/
+#define ZSTD_isError ERR_isError   /* for inlining */
+#define FSE_isError  ERR_isError
+#define HUF_isError  ERR_isError
+
+
 /*_*******************************************************
 *  Memory operations
 **********************************************************/
 static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
 
 
-/*-*************************************
-*  Error Management
-***************************************/
-unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
-
-
 /*-*************************************************************
 *   Context management
 ***************************************************************/
@@ -657,7 +659,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
 }
 
 
-FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
+FORCE_INLINE
+size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
                                 const BYTE** litPtr, const BYTE* const litLimit_8,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -964,31 +967,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
     return dctx->expected;
 }
 
-size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
+size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     /* Sanity check */
     if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
-    ZSTD_checkContinuity(dctx, dst);
+    if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
 
     /* Decompress : frame header; part 1 */
     switch (dctx->stage)
     {
     case ZSTDds_getFrameHeaderSize :
-        {
+        if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
-            if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);   /* impossible */
+        dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
-            dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
+        if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
-            if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
+        memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
-            memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
+        if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
-            if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
+            dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
-                dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
+            dctx->stage = ZSTDds_decodeFrameHeader;
-                dctx->stage = ZSTDds_decodeFrameHeader;
+            return 0;
-                return 0;
-            }
-            dctx->expected = 0;   /* not necessary to copy more */
         }
+        dctx->expected = 0;   /* not necessary to copy more */
+
     case ZSTDds_decodeFrameHeader:
-        {
+        {   size_t result;
-            size_t result;
             memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
             result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
             if (ZSTD_isError(result)) return result;
@@ -997,8 +998,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
             return 0;
         }
     case ZSTDds_decodeBlockHeader:
-        {
+        {   blockProperties_t bp;
-            blockProperties_t bp;
             size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
             if (ZSTD_isError(cBlockSize)) return cBlockSize;
             if (bp.blockType == bt_end) {
@@ -1012,16 +1012,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
             return 0;
         }
     case ZSTDds_decompressBlock:
-        {
+        {   size_t rSize;
-            /* Decompress : block content */
-            size_t rSize;
             switch(dctx->bType)
             {
             case bt_compressed:
-                rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
+                rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
                 break;
             case bt_raw :
-                rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
+                rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
                 break;
             case bt_rle :
                 return ERROR(GENERIC);   /* not yet handled */

lib/legacy/zstd_v05.c

@@ -2885,8 +2885,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
 
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
+    if (cSrcSize >= dstSize) return ERROR(corruption_detected);   /* invalid, or not compressed, but not compressed already dealt with */
-    if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
     if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
 
     /* decoder timing evaluation */

programs/datagen.c

@@ -59,7 +59,6 @@
 typedef BYTE litDistribTable[LTSIZE];
 
 
-
 /*-*******************************************************
 *  Local Functions
 *********************************************************/

programs/datagencli.c

@@ -24,39 +24,17 @@
    - Public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
-/**************************************
+/*-************************************
 *  Includes
 **************************************/
 #include <stdio.h>     /* fprintf, stderr */
+#include "mem.h"
 #include "datagen.h"   /* RDG_generate */
 
 
-/**************************************
+/*-************************************
-*  Basic Types
-**************************************/
-#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
-# include <stdint.h>
-  typedef  uint8_t BYTE;
-  typedef uint16_t U16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
-#else
-  typedef unsigned char       BYTE;
-  typedef unsigned short      U16;
-  typedef unsigned int        U32;
-  typedef   signed int        S32;
-  typedef unsigned long long  U64;
-#endif
-
-
-/**************************************
 *  Constants
 **************************************/
-#ifndef ZSTD_VERSION
-#  define ZSTD_VERSION "r1"
-#endif
-
 #define KB *(1 <<10)
 #define MB *(1 <<20)
 #define GB *(1U<<30)
@@ -66,7 +44,7 @@
 #define COMPRESSIBILITY_DEFAULT 50
 
 
-/**************************************
+/*-************************************
 *  Macros
 **************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
@@ -74,10 +52,10 @@
 static unsigned displayLevel = 2;
 
 
-/*********************************************************
+/*-*******************************************************
 *  Command line
 *********************************************************/
-static int usage(char* programName)
+static int usage(const char* programName)
 {
     DISPLAY( "Compressible data generator\n");
     DISPLAY( "Usage :\n");
@@ -92,29 +70,26 @@ static int usage(char* programName)
 }
 
 
-int main(int argc, char** argv)
+int main(int argc, const char** argv)
 {
     int argNb;
     double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
     double litProba = 0.0;
     U64 size = SIZE_DEFAULT;
     U32 seed = SEED_DEFAULT;
-    char* programName;
+    const char* programName;
 
     /* Check command line */
     programName = argv[0];
-    for(argNb=1; argNb<argc; argNb++)
+    for(argNb=1; argNb<argc; argNb++) {
-    {
+        const char* argument = argv[argNb];
-        char* argument = argv[argNb];
 
         if(!argument) continue;   /* Protection if argument empty */
 
         /* Handle commands. Aggregated commands are allowed */
-        if (*argument=='-')
+        if (*argument=='-') {
-        {
             argument++;
-            while (*argument!=0)
+            while (*argument!=0) {
-            {
                 switch(*argument)
                 {
                 case 'h':
@@ -123,11 +98,7 @@ int main(int argc, char** argv)
                     argument++;
                     size=0;
                     while ((*argument>='0') && (*argument<='9'))
-                    {
+                        size *= 10, size += *argument++ - '0';
-                        size *= 10;
-                        size += *argument - '0';
-                        argument++;
-                    }
                     if (*argument=='K') { size <<= 10; argument++; }
                     if (*argument=='M') { size <<= 20; argument++; }
                     if (*argument=='G') { size <<= 30; argument++; }
@@ -137,21 +108,13 @@ int main(int argc, char** argv)
                     argument++;
                     seed=0;
                     while ((*argument>='0') && (*argument<='9'))
-                    {
+                        seed *= 10, seed += *argument++ - '0';
-                        seed *= 10;
-                        seed += *argument - '0';
-                        argument++;
-                    }
                     break;
                 case 'P':
                     argument++;
                     proba=0.0;
                     while ((*argument>='0') && (*argument<='9'))
-                    {
+                        proba *= 10, proba += *argument++ - '0';
-                        proba *= 10;
-                        proba += *argument - '0';
-                        argument++;
-                    }
                     if (proba>100.) proba=100.;
                     proba /= 100.;
                     break;
@@ -159,11 +122,7 @@ int main(int argc, char** argv)
                     argument++;
                     litProba=0.;
                     while ((*argument>='0') && (*argument<='9'))
-                    {
+                        litProba *= 10, litProba += *argument++ - '0';
-                        litProba *= 10;
-                        litProba += *argument - '0';
-                        argument++;
-                    }
                     if (litProba>100.) litProba=100.;
                     litProba /= 100.;
                     break;
@@ -174,12 +133,9 @@ int main(int argc, char** argv)
                 default:
                     return usage(programName);
                 }
-            }
+    }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
 
-        }
+    DISPLAYLEVEL(4, "Data Generator \n");
-    }
-
-    DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
     DISPLAYLEVEL(3, "Seed = %u \n", seed);
     if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
 

programs/fileio.c

@@ -523,9 +523,9 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
 
     ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
 
-    /* Complete Header loading */
+    /* Header loading (optional, saves one loop) */
-    {   size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded;   /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
+    {   size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded;   /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
-        size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);   /* can be <= toLoad (null string) */
+        size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
         readSize = alreadyLoaded + loadedSize;
     }
 

programs/zbufftest.c

@@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
     }
 
+    /* Byte-by-byte decompression test */
+    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
+    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+    {   size_t r = 1, pIn=0, pOut=0;
+        while (r) {
+            size_t inS = 1;
+            size_t outS = 1;
+            r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
+            pIn += inS;
+            pOut += outS;
+        }
+        readSize = pIn;
+        genSize = pOut;
+    }
+    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
+    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* check regenerated data is byte exact */
+    {   size_t i;
+        DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+        for (i=0; i<CNBufferSize; i++) {
+            if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
+        }
+        DISPLAYLEVEL(4, "OK \n");
+    }
+
 _end:
     ZBUFF_freeCCtx(zc);
     ZBUFF_freeDCtx(zd);