Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

build/VS2005/fuzzer/fuzzer.vcproj

@@ -343,10 +343,22 @@
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>
@@ -393,6 +405,14 @@
 			Filter="h;hpp;hxx;hm;inl;inc;xsd"
 			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
 			>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\bitstream.h"
 				>
@@ -449,6 +469,10 @@
 				RelativePath="..\..\..\lib\common\zstd_internal.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>

build/VS2005/zstdlib/zstdlib.vcproj

@@ -335,6 +335,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -383,6 +391,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -433,6 +445,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -493,6 +513,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_static.h"
 				>

build/VS2008/fuzzer/fuzzer.vcproj

@@ -340,6 +340,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -380,6 +388,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -402,6 +414,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -450,6 +470,10 @@
 				RelativePath="..\..\..\lib\common\zstd_internal.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>

build/VS2008/zstdlib/zstdlib.vcproj

@@ -336,6 +336,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.c"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\entropy_common.c"
 				>
@@ -372,6 +380,10 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
@@ -422,6 +434,14 @@
 				RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\common\pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\..\..\lib\common\threading.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\error_private.h"
 				>
@@ -478,6 +498,10 @@
 				RelativePath="..\..\..\lib\compress\zstd_opt.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\compress\zstdmt_compress.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\common\zstd_static.h"
 				>

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -155,6 +155,8 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
@@ -163,6 +165,7 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
@@ -172,6 +175,8 @@
     <ClCompile Include="..\..\..\tests\fuzzer.c" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\fse.h" />
     <ClInclude Include="..\..\..\lib\common\huf.h" />
     <ClInclude Include="..\..\..\lib\common\xxhash.h" />
@@ -179,6 +184,7 @@
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
     <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
     <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -19,6 +19,8 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
@@ -27,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
@@ -44,6 +47,8 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
@@ -63,6 +68,7 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="libzstd-dll.rc" />

build/VS2010/libzstd/libzstd.vcxproj

@@ -19,6 +19,8 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\pool.c" />
+    <ClCompile Include="..\..\..\lib\common\threading.c" />
     <ClCompile Include="..\..\..\lib\common\entropy_common.c" />
     <ClCompile Include="..\..\..\lib\common\error_private.c" />
     <ClCompile Include="..\..\..\lib\common\xxhash.c" />
@@ -27,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
+    <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
@@ -44,6 +47,8 @@
     <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\..\lib\common\pool.h" />
+    <ClInclude Include="..\..\..\lib\common\threading.h" />
     <ClInclude Include="..\..\..\lib\common\bitstream.h" />
     <ClInclude Include="..\..\..\lib\common\error_private.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
@@ -63,6 +68,7 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+    <ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}</ProjectGuid>

build/cmake/lib/CMakeLists.txt

@@ -59,6 +59,8 @@ SET(Sources
 
 SET(Headers
         ${LIBRARY_DIR}/zstd.h
+        ${LIBRARY_DIR}/common/pool.h
+        ${LIBRARY_DIR}/common/threading.h
         ${LIBRARY_DIR}/common/bitstream.h
         ${LIBRARY_DIR}/common/error_private.h
         ${LIBRARY_DIR}/common/zstd_errors.h

lib/common/pool.h

@@ -9,6 +9,11 @@
 #ifndef POOL_H
 #define POOL_H
 
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
 #include <stddef.h>   /* size_t */
 
 typedef struct POOL_ctx_s POOL_ctx;
@@ -43,4 +48,9 @@ typedef void (*POOL_add_function)(void *, POOL_function, void *);
 */
 void POOL_add(void *ctx, POOL_function function, void *opaque);
 
+
+#if defined (__cplusplus)
+}
+#endif
+
 #endif

lib/compress/zstdmt_compress.c

@@ -21,7 +21,7 @@
 /* ======   Dependencies   ====== */
 #include <stdlib.h>   /* malloc */
 #include <string.h>   /* memcpy */
-#include <pool.h>     /* threadpool */
+#include "pool.h"     /* threadpool */
 #include "threading.h"  /* mutex */
 #include "zstd_internal.h"   /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"

lib/compress/zstdmt_compress.h

@@ -7,6 +7,13 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 
+ #ifndef ZSTDMT_COMPRESS_H
+ #define ZSTDMT_COMPRESS_H
+
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+
 
 /* Note : All prototypes defined in this file shall be considered experimental.
  *        There is no guarantee of API continuity (yet) on any of these prototypes */
@@ -62,3 +69,10 @@ typedef enum {
  * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif   /* ZSTDMT_COMPRESS_H */

lib/dictBuilder/cover.c

@@ -14,11 +14,10 @@
 #include <stdlib.h> /* malloc, free, qsort */
 #include <string.h> /* memset */
 #include <time.h>   /* clock */
-#ifdef ZSTD_PTHREAD
-#include "threading.h"
-#endif
 
 #include "mem.h" /* read */
+#include "pool.h"
+#include "threading.h"
 #include "zstd_internal.h" /* includes zstd.h */
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_STATIC_LINKING_ONLY
@@ -690,11 +689,9 @@ ZDICTLIB_API size_t COVER_trainFromBuffer(
  * compiled with multithreaded support.
  */
 typedef struct COVER_best_s {
-#ifdef ZSTD_PTHREAD
   pthread_mutex_t mutex;
   pthread_cond_t cond;
   size_t liveJobs;
-#endif
   void *dict;
   size_t dictSize;
   COVER_params_t parameters;
@@ -708,11 +705,9 @@ static void COVER_best_init(COVER_best_t *best) {
   if (!best) {
     return;
   }
-#ifdef ZSTD_PTHREAD
   pthread_mutex_init(&best->mutex, NULL);
   pthread_cond_init(&best->cond, NULL);
   best->liveJobs = 0;
-#endif
   best->dict = NULL;
   best->dictSize = 0;
   best->compressedSize = (size_t)-1;
@@ -726,13 +721,11 @@ static void COVER_best_wait(COVER_best_t *best) {
   if (!best) {
     return;
   }
-#ifdef ZSTD_PTHREAD
   pthread_mutex_lock(&best->mutex);
   while (best->liveJobs != 0) {
     pthread_cond_wait(&best->cond, &best->mutex);
   }
   pthread_mutex_unlock(&best->mutex);
-#endif
 }
 
 /**
@@ -746,10 +739,8 @@ static void COVER_best_destroy(COVER_best_t *best) {
   if (best->dict) {
     free(best->dict);
   }
-#ifdef ZSTD_PTHREAD
   pthread_mutex_destroy(&best->mutex);
   pthread_cond_destroy(&best->cond);
-#endif
 }
 
 /**
@@ -760,11 +751,9 @@ static void COVER_best_start(COVER_best_t *best) {
   if (!best) {
     return;
   }
-#ifdef ZSTD_PTHREAD
   pthread_mutex_lock(&best->mutex);
   ++best->liveJobs;
   pthread_mutex_unlock(&best->mutex);
-#endif
 }
 
 /**
@@ -779,12 +768,10 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
     return;
   }
   {
-#ifdef ZSTD_PTHREAD
     size_t liveJobs;
     pthread_mutex_lock(&best->mutex);
     --best->liveJobs;
     liveJobs = best->liveJobs;
-#endif
     /* If the new dictionary is better */
     if (compressedSize < best->compressedSize) {
       /* Allocate space if necessary */
@@ -805,12 +792,10 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
       best->parameters = parameters;
       best->compressedSize = compressedSize;
     }
-#ifdef ZSTD_PTHREAD
     pthread_mutex_unlock(&best->mutex);
     if (liveJobs == 0) {
       pthread_cond_broadcast(&best->cond);
     }
-#endif
   }
 }
 
@@ -928,11 +913,12 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
                                                   unsigned nbSamples,
                                                   COVER_params_t *parameters) {
   /* constants */
+  const unsigned nbThreads = parameters->nbThreads;
   const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
   const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d;
   const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k;
   const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k;
-  const unsigned kSteps = parameters->steps == 0 ? 256 : parameters->steps;
+  const unsigned kSteps = parameters->steps == 0 ? 32 : parameters->steps;
   const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
   const unsigned kIterations =
       (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
@@ -942,6 +928,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
   unsigned d;
   unsigned k;
   COVER_best_t best;
+  POOL_ctx *pool = NULL;
   /* Checks */
   if (kMinK < kMaxD || kMaxK < kMinK) {
     LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
@@ -956,6 +943,12 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
                  ZDICT_DICTSIZE_MIN);
     return ERROR(dstSize_tooSmall);
   }
+  if (nbThreads > 1) {
+    pool = POOL_create(nbThreads, 1);
+    if (!pool) {
+      return ERROR(memory_allocation);
+    }
+  }
   /* Initialization */
   COVER_best_init(&best);
   /* Turn down global display level to clean up display at level 2 and below */
@@ -998,7 +991,11 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
       }
       /* Call the function and pass ownership of data to it */
       COVER_best_start(&best);
+      if (pool) {
+        POOL_add(pool, &COVER_tryParameters, data);
+      } else {
         COVER_tryParameters(data);
+      }
       /* Print status */
       LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%%       ",
                          (U32)((iteration * 100) / kIterations));
@@ -1018,6 +1015,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
     *parameters = best.parameters;
     memcpy(dictBuffer, best.dict, dictSize);
     COVER_best_destroy(&best);
+    POOL_free(pool);
     return dictSize;
   }
 }

lib/dictBuilder/zdict.h

@@ -93,8 +93,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dict
 typedef struct {
     unsigned k;                  /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
     unsigned d;                  /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
-    unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (256) : Higher means more parameters checked */
+    unsigned steps;              /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
 
+    unsigned nbThreads;          /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
     unsigned notificationLevel;  /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
     unsigned dictID;             /* 0 means auto mode (32-bits random value); other : force dictID value */
     int      compressionLevel;   /* 0 means default; target a specific zstd compression level */
@@ -132,7 +133,7 @@ ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
     @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
               or an error code, which can be tested with ZDICT_isError().
               On success `*parameters` contains the parameters selected.
-    Note : COVER_optimizeTrainFromBuffer() requires about 9 bytes of memory for each input byte.
+    Note : COVER_optimizeTrainFromBuffer() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
 */
 ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
                                      const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,

lib/legacy/zstd_v04.c

@@ -3012,20 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
     /* Literal length */
     litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
     prevOffset = litLength ? seq->offset : seqState->prevOffset;
-    if (litLength == MaxLL)
+    if (litLength == MaxLL) {
-    {
         U32 add = *dumps++;
         if (add < 255) litLength += add;
         else {
-            litLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+            litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
             dumps += 3;
         }
-        if (dumps >= de) { dumps = de-1; litLength = MaxLL+255; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { litLength = MaxLL+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
 
     /* Offset */
-    {
+    {   static const U32 offsetPrefix[MaxOff+1] = {
-        static const U32 offsetPrefix[MaxOff+1] = {
                 1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
                 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
                 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3046,10 +3045,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
         U32 add = *dumps++;
         if (add < 255) matchLength += add;
         else {
-            matchLength = MEM_readLE32(dumps) & 0xFFFFFF;  /* no pb : dumps is always followed by seq tables > 1 byte */
+            matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
             dumps += 3;
         }
-        if (dumps >= de) { dumps = de-1; matchLength = MaxML+255; }   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { matchLength = MaxML+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
     matchLength += MINMATCH;
 

lib/legacy/zstd_v05.c

@@ -3230,7 +3230,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
             if (litLength&1) litLength>>=1, dumps += 3;
             else litLength = (U16)(litLength)>>1, dumps += 2;
         }
-        if (dumps >= de) { dumps = de-1; litLength = MaxLL+255; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { litLength = MaxLL+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
 
     /* Offset */
@@ -3263,7 +3264,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
             if (matchLength&1) matchLength>>=1, dumps += 3;
             else matchLength = (U16)(matchLength)>>1, dumps += 2;
         }
-        if (dumps >= de) { dumps = de-1; matchLength = MaxML+255; }   /* late correction, to avoid read overflow (data is now corrupted anyway) */
+        if (dumps > de) { matchLength = MaxML+255; }  /* late correction, to avoid using uninitialized memory */
+        if (dumps >= de) { dumps = de-1; }  /* late correction, to avoid read overflow (data is now corrupted anyway) */
     }
     matchLength += MINMATCH;
 

programs/zstdcli.c

@@ -577,6 +577,7 @@ int main(int argCount, const char* argv[])
     if (operation==zom_train) {
 #ifndef ZSTD_NODICT
         if (cover) {
+            coverParams.nbThreads = nbThreads;
             coverParams.compressionLevel = dictCLevel;
             coverParams.notificationLevel = displayLevel;
             coverParams.dictID = dictID;