diff --git a/Makefile b/Makefile index 55c24530..25a6c242 100644 --- a/Makefile +++ b/Makefile @@ -57,8 +57,8 @@ all32: $(MAKE) -C $(PRGDIR) zstd32 $(MAKE) -C $(TESTDIR) all32 -.PHONY: lib lib-release -lib lib-release : +.PHONY: lib lib-release lib-mt lib-nomt +lib lib-release lib-mt lib-nomt: $(Q)$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -151,7 +151,6 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) EGREP_OPTIONS ?= @@ -357,12 +356,13 @@ lz4install: endif +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release + ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON endif - #------------------------------------------------------------------------ # target specific tests #------------------------------------------------------------------------ diff --git a/appveyor.yml b/appveyor.yml index 24437be9..c6ab7868 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -256,7 +256,6 @@ - if [%HOST%]==[cygwin] ( ECHO Installing Cygwin Packages && C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^ - gcc-g++,^ gcc,^ cmake,^ make @@ -279,8 +278,8 @@ C:\cygwin64\bin\bash --login -c " set -e; cd build/cmake; - CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .; - make -j4; + CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .; + make VERBOSE=1 -j; ctest -V -L Medium; " ) diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index f7b6e908..27acd41b 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -59,7 +59,7 @@ target_link_libraries(datagen libzstd_static) add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c) set_property(TARGET fullbench APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(fullbench libzstd_static) -add_test(NAME fullbench COMMAND fullbench) +add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS}) # # fuzzer diff --git a/lib/Makefile b/lib/Makefile index 2ce696b5..e38c11fc 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -8,6 +8,9 @@ # You may select, at your option, one of the above-listed licenses. # ################################################################ +# Note: by default, the static library is built single-threaded and dynamic library is built +# multi-threaded. It is possible to force multi or single threaded builds by appending +# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded). .PHONY: default default: lib-release @@ -68,6 +71,10 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) +CPPFLAGS_DYNLIB = -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded +LDFLAGS_DYNLIB = -pthread +CPPFLAGS_STATLIB = # static library build defaults to single-threaded + HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) @@ -224,6 +231,7 @@ all: lib .PHONY: libzstd.a # must be run every time +libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB) ifndef BUILD_DIR # determine BUILD_DIR from compilation flags @@ -240,7 +248,10 @@ ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) $(ZSTD_STATLIB): ARFLAGS = rcs $(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR) $(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ) - @echo compiling static library + # Check for multithread flag at target execution time + $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ + @echo compiling multi-threaded static library $(LIBVER),\ + @echo compiling single-threaded static library $(LIBVER)) $(AR) $(ARFLAGS) $@ $^ libzstd.a: $(ZSTD_STATLIB) @@ -259,8 +270,9 @@ else # not Windows LIBZSTD = libzstd.$(SHARED_EXT_VER) .PHONY: $(LIBZSTD) # must be run every time -$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden -$(LIBZSTD): LDFLAGS += -shared +$(LIBZSTD): CPPFLAGS += $(CPPFLAGS_DYNLIB) +$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden +$(LIBZSTD): LDFLAGS += -shared $(LDFLAGS_DYNLIB) ifndef BUILD_DIR # determine BUILD_DIR from compilation flags @@ -277,7 +289,10 @@ ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) $(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR) $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) - @echo compiling dynamic library $(LIBVER) +# Check for multithread flag at target execution time + $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ + @echo compiling multi-threaded dynamic library $(LIBVER),\ + @echo compiling single-threaded dynamic library $(LIBVER)) $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) @@ -299,10 +314,17 @@ lib : libzstd.a libzstd # note : do not define lib-mt or lib-release as .PHONY # make does not consider implicit pattern rule for .PHONY target -%-mt : CPPFLAGS += -DZSTD_MULTITHREAD -%-mt : LDFLAGS += -pthread +%-mt : CPPFLAGS_DYNLIB := -DZSTD_MULTITHREAD +%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD +%-mt : LDFLAGS_DYNLIB := -pthread %-mt : % - @echo multi-threading build completed + @echo multi-threaded build completed + +%-nomt : CPPFLAGS_DYNLIB := +%-nomt : LDFLAGS_DYNLIB := +%-nomt : CPPFLAGS_STATLIB := +%-nomt : % + @echo single-threaded build completed %-release : DEBUGFLAGS := %-release : % diff --git a/lib/README.md b/lib/README.md index cbf3f01b..f781ac57 100644 --- a/lib/README.md +++ b/lib/README.md @@ -19,12 +19,16 @@ The scope can be reduced on demand (see paragraph _modular build_). #### Multithreading support -Multithreading is disabled by default when building with `make`. +When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons). + Enabling multithreading requires 2 conditions : - set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) - for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) -Both conditions are automatically applied when invoking `make lib-mt` target. +For convenience, we provide a build target to generate multi and single threaded libraries: +- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`. +- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`. +- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`. When linking a POSIX program with a multithreaded version of `libzstd`, note that it's necessary to invoke the `-pthread` flag during link stage. diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 46147a24..b4b2b6f7 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -42,6 +42,7 @@ #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ +#include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ /*-************************************ @@ -335,6 +336,126 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, } } +#ifdef ZSTD_MULTITHREAD +typedef struct { + ZSTD_CCtx* cctx; + ZSTD_threadPool* pool; + void* const CNBuffer; + size_t CNBuffSize; + void* const compressedBuffer; + size_t compressedBufferSize; + void* const decodedBuffer; + int err; +} threadPoolTests_compressionJob_payload; + +static void* threadPoolTests_compressionJob(void* payload) { + threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload; + size_t cSize; + if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1; + cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize); + if (ZSTD_isError(cSize)) args->err = 1; + if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1; + return payload; +} + +static int threadPoolTests(void) { + int testResult = 0; + size_t err; + + size_t const CNBuffSize = 5 MB; + void* const CNBuffer = malloc(CNBuffSize); + size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); + void* const compressedBuffer = malloc(compressedBufferSize); + void* const decodedBuffer = malloc(CNBuffSize); + + size_t const kPoolNumThreads = 8; + + RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); + + DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: "); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); + + size_t nbThreads = 1; + for (; nbThreads <= kPoolNumThreads; ++nbThreads) { + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads); + err = ZSTD_CCtx_refThreadPool(cctx, pool); + if (ZSTD_isError(err)) { + DISPLAYLEVEL(3, "refThreadPool error!\n"); + ZSTD_freeCCtx(cctx); + goto _output_error; + } + err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + if (ZSTD_isError(err)) { + DISPLAYLEVEL(3, "Compression error!\n"); + ZSTD_freeCCtx(cctx); + goto _output_error; + } + err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err); + if (ZSTD_isError(err)) { + DISPLAYLEVEL(3, "Decompression error!\n"); + ZSTD_freeCCtx(cctx); + goto _output_error; + } + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeThreadPool(pool); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: "); + { + void* const decodedBuffer2 = malloc(CNBuffSize); + void* const compressedBuffer2 = malloc(compressedBufferSize); + ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); + ZSTD_CCtx* cctx1 = ZSTD_createCCtx(); + ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); + + ZSTD_pthread_t t1; + ZSTD_pthread_t t2; + threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize, + compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */}; + threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize, + compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */}; + + ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2); + ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2); + ZSTD_CCtx_refThreadPool(cctx1, pool); + ZSTD_CCtx_refThreadPool(cctx2, pool); + + ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1); + ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2); + ZSTD_pthread_join(t1, NULL); + ZSTD_pthread_join(t2, NULL); + + assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize)); + free(decodedBuffer2); + free(compressedBuffer2); + + ZSTD_freeThreadPool(pool); + ZSTD_freeCCtx(cctx1); + ZSTD_freeCCtx(cctx2); + + if (p1.err || p2.err) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + +_end: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected in Unit tests ! \n"); + goto _end; +} +#endif /* ZSTD_MULTITHREAD */ + /*============================================= * Unit tests =============================================*/ @@ -3292,7 +3413,16 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); -#endif + DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) + { + int const threadPoolTestResult = threadPoolTests(); + if (threadPoolTestResult) { + goto _output_error; + } + } + DISPLAYLEVEL(3, "thread pool tests OK \n"); + +#endif /* ZSTD_MULTITHREAD */ _end: free(CNBuffer);