Merge branch 'dev' of github.com:facebook/zstd into dev

appveyor.yml

@@ -45,7 +45,13 @@ build_script:
       ECHO make %CLANG_PARAMS% &&
       make %CLANG_PARAMS% &&
       COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
-      make clean
+      make clean &&
+      ECHO *** &&
+      ECHO *** Building pzstd for %PLATFORM% &&
+      ECHO *** &&
+      ECHO make -C contrib\pzstd pzstd &&
+      make -C contrib\pzstd pzstd &&
+      make -C contrib\pzstd clean
     )
   - if [%COMPILER%]==[gcc] (
       ECHO *** &&

contrib/pzstd/Options.cpp

@@ -10,6 +10,7 @@
 
 #include <cstdio>
 #include <cstring>
+#include <thread>
 
 namespace pzstd {
 
@@ -103,6 +104,7 @@ bool Options::parse(int argc, const char** argv) {
         numThreads = parseUnsigned(argv[i]);
         if (numThreads == 0) {
           std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n");
+          return false;
         }
         break;
       case 'p':
@@ -169,13 +171,18 @@ bool Options::parse(int argc, const char** argv) {
     if (compressionLevel > maxCLevel) {
       std::fprintf(
           stderr, "Invalid compression level %u.\n", compressionLevel);
+      return false;
     }
   }
   // Check that numThreads is set
   if (numThreads == 0) {
-    std::fprintf(stderr, "Invalid arguments: # of threads not specified.\n");
+    numThreads = std::thread::hardware_concurrency();
+    if (numThreads == 0) {
+      std::fprintf(stderr, "Invalid arguments: # of threads not specified "
+                           "and unable to determine hardware concurrency.\n");
       return false;
     }
+  }
   return true;
 }
 }

contrib/pzstd/Pzstd.cpp

@@ -34,7 +34,7 @@ using std::size_t;
 size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
   // Open the input file and attempt to determine its size
   FILE* inputFd = stdin;
-  size_t inputSize = 0;
+  std::uintmax_t inputSize = 0;
   if (options.inputFile != "-") {
     inputFd = std::fopen(options.inputFile.c_str(), "rb");
     if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
@@ -155,7 +155,7 @@ static void compress(
     ZSTD_parameters parameters) {
   auto guard = makeScopeGuard([&] { out->finish(); });
   // Initialize the CCtx
-  std::unique_ptr<ZSTD_CStream, size_t (&)(ZSTD_CStream*)> ctx(
+  std::unique_ptr<ZSTD_CStream, size_t (*)(ZSTD_CStream*)> ctx(
       ZSTD_createCStream(), ZSTD_freeCStream);
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
     return;
@@ -217,14 +217,17 @@ static void compress(
  * @param numThreads The number of threads available to run compression jobs on
  * @param params     The zstd parameters to be used for compression
  */
-static size_t
+static size_t calculateStep(
-calculateStep(size_t size, size_t numThreads, const ZSTD_parameters& params) {
+    std::uintmax_t size,
-  size_t step = 1ul << (params.cParams.windowLog + 2);
+    size_t numThreads,
+    const ZSTD_parameters &params) {
+  size_t step = size_t{1} << (params.cParams.windowLog + 2);
   // If file size is known, see if a smaller step will spread work more evenly
   if (size != 0) {
-    size_t newStep = size / numThreads;
+    const std::uintmax_t newStep = size / std::uintmax_t{numThreads};
-    if (newStep != 0) {
+    if (newStep != 0 &&
-      step = std::min(step, newStep);
+        newStep <= std::uintmax_t{std::numeric_limits<size_t>::max()}) {
+      step = std::min(step, size_t{newStep});
     }
   }
   return step;
@@ -268,7 +271,7 @@ void asyncCompressChunks(
     WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
     ThreadPool& executor,
     FILE* fd,
-    size_t size,
+    std::uintmax_t size,
     size_t numThreads,
     ZSTD_parameters params) {
   auto chunksGuard = makeScopeGuard([&] { chunks.finish(); });
@@ -311,7 +314,7 @@ static void decompress(
     std::shared_ptr<BufferWorkQueue> out) {
   auto guard = makeScopeGuard([&] { out->finish(); });
   // Initialize the DCtx
-  std::unique_ptr<ZSTD_DStream, size_t (&)(ZSTD_DStream*)> ctx(
+  std::unique_ptr<ZSTD_DStream, size_t (*)(ZSTD_DStream*)> ctx(
       ZSTD_createDStream(), ZSTD_freeDStream);
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
     return;

contrib/pzstd/Pzstd.h

@@ -19,6 +19,7 @@
 #undef ZSTD_STATIC_LINKING_ONLY
 
 #include <cstddef>
+#include <cstdint>
 #include <memory>
 
 namespace pzstd {
@@ -52,7 +53,7 @@ void asyncCompressChunks(
     WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
     ThreadPool& executor,
     FILE* fd,
-    std::size_t size,
+    std::uintmax_t size,
     std::size_t numThreads,
     ZSTD_parameters parameters);
 

contrib/pzstd/test/OptionsTest.cpp

@@ -118,11 +118,11 @@ TEST(Options, ValidInputs) {
   }
 }
 
-TEST(Options, BadNumThreads) {
+TEST(Options, NumThreads) {
   {
     Options options;
     std::array<const char*, 3> args = {{nullptr, "-o", "-"}};
-    EXPECT_FALSE(options.parse(args.size(), args.data()));
+    EXPECT_TRUE(options.parse(args.size(), args.data()));
   }
   {
     Options options;

contrib/pzstd/utils/FileSystem.h

@@ -11,6 +11,7 @@
 #include "utils/Range.h"
 
 #include <sys/stat.h>
+#include <cerrno>
 #include <cstdint>
 #include <system_error>
 
@@ -20,12 +21,21 @@
 
 namespace pzstd {
 
-using file_status = struct stat;
+#if defined(_MSC_VER)
+using file_status = struct ::_stat64;
+#else
+using file_status = struct ::stat;
+#endif
 
 /// http://en.cppreference.com/w/cpp/filesystem/status
 inline file_status status(StringPiece path, std::error_code& ec) noexcept {
   file_status status;
-  if (stat(path.data(), &status)) {
+#if defined(_MSC_VER)
+  const auto error = ::_stat64(path.data(), &status);
+#else
+  const auto error = ::stat(path.data(), &status);
+#endif
+  if (error) {
     ec.assign(errno, std::generic_category());
   } else {
     ec.clear();
@@ -35,7 +45,13 @@ inline file_status status(StringPiece path, std::error_code& ec) noexcept {
 
 /// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
 inline bool is_regular_file(file_status status) noexcept {
+#if defined(S_ISREG)
   return S_ISREG(status.st_mode);
+#elif !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG)
+  return (status.st_mode & S_IFMT) == S_IFREG;
+#else
+  static_assert(false, "No POSIX stat() support.");
+#endif
 }
 
 /// http://en.cppreference.com/w/cpp/filesystem/is_regular_file

lib/zstd.h

@@ -299,8 +299,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp
 #define ZSTD_HASHLOG3_MAX      17
 #define ZSTD_SEARCHLOG_MAX    (ZSTD_WINDOWLOG_MAX-1)
 #define ZSTD_SEARCHLOG_MIN      1
-#define ZSTD_SEARCHLENGTH_MAX   7
+#define ZSTD_SEARCHLENGTH_MAX   7   /* only for ZSTD_fast, other strategies are limited to 6 */
-#define ZSTD_SEARCHLENGTH_MIN   3
+#define ZSTD_SEARCHLENGTH_MIN   3   /* only for ZSTD_btopt, other strategies are limited to 4 */
 #define ZSTD_TARGETLENGTH_MIN   4
 #define ZSTD_TARGETLENGTH_MAX 999
 

programs/fileio.c

@@ -732,7 +732,10 @@ static int FIO_decompressDstFile(dRess_t ress,
     result = FIO_decompressSrcFile(ress, srcFileName);
 
     if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
-    if (result != 0) if (remove(dstFileName)) result=1;   /* don't do anything if remove fails */
+    if ( (result != 0)
+       && strcmp(dstFileName, nulmark)  /* special case : don't remove() /dev/null (#316) */
+       && remove(dstFileName) )
+        result=1;   /* don't do anything special if remove fails */
     return result;
 }