Merge branch 'dev' of github.com:facebook/zstd into dev
commit
5c6d244973
|
@ -45,7 +45,13 @@ build_script:
|
|||
ECHO make %CLANG_PARAMS% &&
|
||||
make %CLANG_PARAMS% &&
|
||||
COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
|
||||
make clean
|
||||
make clean &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building pzstd for %PLATFORM% &&
|
||||
ECHO *** &&
|
||||
ECHO make -C contrib\pzstd pzstd &&
|
||||
make -C contrib\pzstd pzstd &&
|
||||
make -C contrib\pzstd clean
|
||||
)
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
ECHO *** &&
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <thread>
|
||||
|
||||
namespace pzstd {
|
||||
|
||||
|
@ -103,6 +104,7 @@ bool Options::parse(int argc, const char** argv) {
|
|||
numThreads = parseUnsigned(argv[i]);
|
||||
if (numThreads == 0) {
|
||||
std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n");
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case 'p':
|
||||
|
@ -169,12 +171,17 @@ bool Options::parse(int argc, const char** argv) {
|
|||
if (compressionLevel > maxCLevel) {
|
||||
std::fprintf(
|
||||
stderr, "Invalid compression level %u.\n", compressionLevel);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Check that numThreads is set
|
||||
if (numThreads == 0) {
|
||||
std::fprintf(stderr, "Invalid arguments: # of threads not specified.\n");
|
||||
return false;
|
||||
numThreads = std::thread::hardware_concurrency();
|
||||
if (numThreads == 0) {
|
||||
std::fprintf(stderr, "Invalid arguments: # of threads not specified "
|
||||
"and unable to determine hardware concurrency.\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -34,7 +34,7 @@ using std::size_t;
|
|||
size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
|
||||
// Open the input file and attempt to determine its size
|
||||
FILE* inputFd = stdin;
|
||||
size_t inputSize = 0;
|
||||
std::uintmax_t inputSize = 0;
|
||||
if (options.inputFile != "-") {
|
||||
inputFd = std::fopen(options.inputFile.c_str(), "rb");
|
||||
if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
|
||||
|
@ -155,7 +155,7 @@ static void compress(
|
|||
ZSTD_parameters parameters) {
|
||||
auto guard = makeScopeGuard([&] { out->finish(); });
|
||||
// Initialize the CCtx
|
||||
std::unique_ptr<ZSTD_CStream, size_t (&)(ZSTD_CStream*)> ctx(
|
||||
std::unique_ptr<ZSTD_CStream, size_t (*)(ZSTD_CStream*)> ctx(
|
||||
ZSTD_createCStream(), ZSTD_freeCStream);
|
||||
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
|
||||
return;
|
||||
|
@ -217,14 +217,17 @@ static void compress(
|
|||
* @param numThreads The number of threads available to run compression jobs on
|
||||
* @param params The zstd parameters to be used for compression
|
||||
*/
|
||||
static size_t
|
||||
calculateStep(size_t size, size_t numThreads, const ZSTD_parameters& params) {
|
||||
size_t step = 1ul << (params.cParams.windowLog + 2);
|
||||
static size_t calculateStep(
|
||||
std::uintmax_t size,
|
||||
size_t numThreads,
|
||||
const ZSTD_parameters ¶ms) {
|
||||
size_t step = size_t{1} << (params.cParams.windowLog + 2);
|
||||
// If file size is known, see if a smaller step will spread work more evenly
|
||||
if (size != 0) {
|
||||
size_t newStep = size / numThreads;
|
||||
if (newStep != 0) {
|
||||
step = std::min(step, newStep);
|
||||
const std::uintmax_t newStep = size / std::uintmax_t{numThreads};
|
||||
if (newStep != 0 &&
|
||||
newStep <= std::uintmax_t{std::numeric_limits<size_t>::max()}) {
|
||||
step = std::min(step, size_t{newStep});
|
||||
}
|
||||
}
|
||||
return step;
|
||||
|
@ -268,7 +271,7 @@ void asyncCompressChunks(
|
|||
WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
|
||||
ThreadPool& executor,
|
||||
FILE* fd,
|
||||
size_t size,
|
||||
std::uintmax_t size,
|
||||
size_t numThreads,
|
||||
ZSTD_parameters params) {
|
||||
auto chunksGuard = makeScopeGuard([&] { chunks.finish(); });
|
||||
|
@ -311,7 +314,7 @@ static void decompress(
|
|||
std::shared_ptr<BufferWorkQueue> out) {
|
||||
auto guard = makeScopeGuard([&] { out->finish(); });
|
||||
// Initialize the DCtx
|
||||
std::unique_ptr<ZSTD_DStream, size_t (&)(ZSTD_DStream*)> ctx(
|
||||
std::unique_ptr<ZSTD_DStream, size_t (*)(ZSTD_DStream*)> ctx(
|
||||
ZSTD_createDStream(), ZSTD_freeDStream);
|
||||
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
|
||||
return;
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
#undef ZSTD_STATIC_LINKING_ONLY
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
namespace pzstd {
|
||||
|
@ -52,7 +53,7 @@ void asyncCompressChunks(
|
|||
WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
|
||||
ThreadPool& executor,
|
||||
FILE* fd,
|
||||
std::size_t size,
|
||||
std::uintmax_t size,
|
||||
std::size_t numThreads,
|
||||
ZSTD_parameters parameters);
|
||||
|
||||
|
|
|
@ -118,11 +118,11 @@ TEST(Options, ValidInputs) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST(Options, BadNumThreads) {
|
||||
TEST(Options, NumThreads) {
|
||||
{
|
||||
Options options;
|
||||
std::array<const char*, 3> args = {{nullptr, "-o", "-"}};
|
||||
EXPECT_FALSE(options.parse(args.size(), args.data()));
|
||||
EXPECT_TRUE(options.parse(args.size(), args.data()));
|
||||
}
|
||||
{
|
||||
Options options;
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
#include "utils/Range.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <cerrno>
|
||||
#include <cstdint>
|
||||
#include <system_error>
|
||||
|
||||
|
@ -20,12 +21,21 @@
|
|||
|
||||
namespace pzstd {
|
||||
|
||||
using file_status = struct stat;
|
||||
#if defined(_MSC_VER)
|
||||
using file_status = struct ::_stat64;
|
||||
#else
|
||||
using file_status = struct ::stat;
|
||||
#endif
|
||||
|
||||
/// http://en.cppreference.com/w/cpp/filesystem/status
|
||||
inline file_status status(StringPiece path, std::error_code& ec) noexcept {
|
||||
file_status status;
|
||||
if (stat(path.data(), &status)) {
|
||||
#if defined(_MSC_VER)
|
||||
const auto error = ::_stat64(path.data(), &status);
|
||||
#else
|
||||
const auto error = ::stat(path.data(), &status);
|
||||
#endif
|
||||
if (error) {
|
||||
ec.assign(errno, std::generic_category());
|
||||
} else {
|
||||
ec.clear();
|
||||
|
@ -35,7 +45,13 @@ inline file_status status(StringPiece path, std::error_code& ec) noexcept {
|
|||
|
||||
/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
|
||||
inline bool is_regular_file(file_status status) noexcept {
|
||||
#if defined(S_ISREG)
|
||||
return S_ISREG(status.st_mode);
|
||||
#elif !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG)
|
||||
return (status.st_mode & S_IFMT) == S_IFREG;
|
||||
#else
|
||||
static_assert(false, "No POSIX stat() support.");
|
||||
#endif
|
||||
}
|
||||
|
||||
/// http://en.cppreference.com/w/cpp/filesystem/is_regular_file
|
||||
|
|
|
@ -299,8 +299,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp
|
|||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
|
||||
#define ZSTD_TARGETLENGTH_MIN 4
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
|
||||
|
|
|
@ -732,7 +732,10 @@ static int FIO_decompressDstFile(dRess_t ress,
|
|||
result = FIO_decompressSrcFile(ress, srcFileName);
|
||||
|
||||
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
|
||||
if (result != 0) if (remove(dstFileName)) result=1; /* don't do anything if remove fails */
|
||||
if ( (result != 0)
|
||||
&& strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */
|
||||
&& remove(dstFileName) )
|
||||
result=1; /* don't do anything special if remove fails */
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue