From 9648bf027bee71db8f2cad1250c6c65730b0f2a1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 20 Dec 2020 17:10:57 -0800 Subject: [PATCH 1/7] try to keep libzstd.a "as is" once created to be compatible with scenarios such as `make -j allmost` --- Makefile | 7 +++---- lib/Makefile | 10 +++++----- programs/Makefile | 2 +- tests/Makefile | 9 ++++----- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 2832fb47..f5bc7044 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ allmost: allzstd zlibwrapper # skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd -allzstd: lib-all +allzstd: lib $(Q)$(MAKE) -C $(PRGDIR) all $(Q)$(MAKE) -C $(TESTDIR) all @@ -57,9 +57,8 @@ all32: $(MAKE) -C $(PRGDIR) zstd32 $(MAKE) -C $(TESTDIR) all32 -.PHONY: lib lib-release libzstd.a -lib-all : lib -lib lib-release lib-all : +.PHONY: lib lib-release +lib lib-release : $(Q)$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release diff --git a/lib/Makefile b/lib/Makefile index 869d7663..c9bd4244 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -215,13 +215,10 @@ SET_CACHE_DIRECTORY = \ LDFLAGS="$(LDFLAGS)" -.PHONY: lib-all all clean install uninstall - -# alias -lib-all: all - +.PHONY: all all: lib + .PHONY: libzstd.a # must be run every time ifndef BUILD_DIR @@ -339,6 +336,7 @@ libzstd-nomt: $(ZSTD_NOMT_FILES) @echo files : $(ZSTD_NOMT_FILES) $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ +.PHONY: clean clean: $(RM) -r *.dSYM # macOS-specific $(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc @@ -407,6 +405,7 @@ libzstd.pc: libzstd.pc.in -e 's|@VERSION@|$(VERSION)|' \ $< >$@ +.PHONY: install install: install-pc install-static install-shared install-includes @echo zstd static and shared library installed @@ -437,6 +436,7 @@ install-includes: $(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR) $(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR) +.PHONY: uninstall uninstall: $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) diff --git a/programs/Makefile b/programs/Makefile index 8641d0ee..e932c129 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -229,7 +229,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ) @echo "$(LZMA_MSG)" @echo "$(LZ4_MSG)" @echo LINK $@ - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) + $(LINK.o) $^ -o $(LDLIBS) $@$(EXT) ifeq ($(HAVE_HASH),1) SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ") diff --git a/tests/Makefile b/tests/Makefile index 42bc353c..1c667cf8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -38,8 +38,8 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(DEBUGFLAGS) +CPPFLAGS += $(MOREFLAGS) ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c @@ -107,7 +107,6 @@ libzstd : %-dll : libzstd %-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd -.PHONY: $(ZSTDDIR)/libzstd.a $(ZSTDDIR)/libzstd.a : $(MAKE) -C $(ZSTDDIR) libzstd.a @@ -146,7 +145,7 @@ fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR # note : broken : requires symbols unavailable from dynamic library fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll - $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) + $(LINK.c) $^ $(LDLIBS) -o $@$(EXT) fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) fuzzer : LDFLAGS += $(MULTITHREAD_LD) @@ -165,7 +164,7 @@ zbufftest zbufftest32 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest zbufftest32 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings zbufftest32 : CFLAGS += -m32 zbufftest zbufftest32 : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) From f9884036c2ce6cca05d5d253ba9695968a5c8da2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 20 Dec 2020 17:19:23 -0800 Subject: [PATCH 2/7] fixed zstd recipe --- programs/Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index e932c129..86c4610c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -196,7 +196,8 @@ SET_CACHE_DIRECTORY = \ BUILD_DIR=obj/$(HASH_DIR) \ CPPFLAGS="$(CPPFLAGS)" \ CFLAGS="$(CFLAGS)" \ - LDFLAGS="$(LDFLAGS)" + LDFLAGS="$(LDFLAGS)" \ + LDLIBS="$(LDLIBS)" .PHONY: all @@ -207,7 +208,8 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-di .PHONY: zstd # must always be run zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) -zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) +zstd : LDFLAGS += $(THREAD_LD) $(DEBUGFLAGS_LD) +zstd : LDLIBS += $(ZLIBLD) $(LZMALD) $(LZ4LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) @@ -229,7 +231,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ) @echo "$(LZMA_MSG)" @echo "$(LZ4_MSG)" @echo LINK $@ - $(LINK.o) $^ -o $(LDLIBS) $@$(EXT) + $(LINK.o) $^ $(LDLIBS) -o $@$(EXT) ifeq ($(HAVE_HASH),1) SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ") From 9a9d3f76c48f8223cf4a5cd1279ad29615a5c2e0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 20 Dec 2020 17:53:04 -0800 Subject: [PATCH 3/7] fixed zstd+sanitizer build --- .github/workflows/generic-dev.yml | 2 +- programs/Makefile | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 8882fcc0..85987ef3 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -71,7 +71,7 @@ jobs: make gcc8install CC=gcc-8 CFLAGS="-Werror" make -j all make clean - CC=gcc-8 make -j uasan-test-zstd Date: Sun, 20 Dec 2020 18:29:36 -0800 Subject: [PATCH 4/7] updated clang+msan test --- .github/workflows/generic-dev.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 85987ef3..1cda7ddc 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -84,17 +84,15 @@ jobs: make clean CC=gcc-6 make -j uasan-test-zstd32 - clang-38-msan-testzstd: - runs-on: ubuntu-16.04 # fails on 18.04 + clang-msan-testzstd: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: clang-3.8 + MSan + Test Zstd + - name: clang + MSan + Test Zstd run: | - # make clang38install (doesn't work) - sudo apt-add-repository "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8 main" sudo apt-get update - sudo apt-get install clang-3.8 - CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 + sudo apt-get install clang + CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1 # Note : external libraries must be turned off when using MSAN tests, # because they are not msan-instrumented, From 8233f55df8435b2d5d429da5b0fabc2e43533d32 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 20 Dec 2020 21:31:58 -0800 Subject: [PATCH 5/7] verbose cross-compile tests on circleci for better diagnosis --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9738c1bd..4529743e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,9 +31,9 @@ jobs: command: | make gnu90build; make clean make gnu99build; make clean - make ppc64build; make clean - make ppcbuild ; make clean - make armbuild ; make clean + make ppc64build V=1; make clean + make ppcbuild V=1; make clean + make armbuild V=1; make clean make -C tests test-legacy test-longmatch; make clean make -C lib libzstd-nomt; make clean # This step is only run on release tags. From 585196353d8d243c5da76318db06cfbe7c4e194b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 20 Dec 2020 21:44:40 -0800 Subject: [PATCH 6/7] fix ppc64 build on circleci --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f5bc7044..c7d33b28 100644 --- a/Makefile +++ b/Makefile @@ -224,10 +224,10 @@ aarch64build: clean CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd ppcbuild: clean - CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd + CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd ppc64build: clean - CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd + CC=powerpc-linux-gnu-gcc CFLAGS=-Werror TARGET_ARCH=-m64 $(MAKE) -j allzstd armfuzz: clean CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest From f2ac2b7bcf6294f1c3a8d62b5f2ca78cfc6fb980 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 21 Dec 2020 15:43:14 -0800 Subject: [PATCH 7/7] try to fix cross-compiler tests --- Makefile | 2 +- appveyor.yml | 4 ++-- programs/Makefile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c7d33b28..bf9cf3e3 100644 --- a/Makefile +++ b/Makefile @@ -227,7 +227,7 @@ ppcbuild: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd ppc64build: clean - CC=powerpc-linux-gnu-gcc CFLAGS=-Werror TARGET_ARCH=-m64 $(MAKE) -j allzstd + CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd armfuzz: clean CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest diff --git a/appveyor.yml b/appveyor.yml index 0e872557..17215931 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ - COMPILER: "clang" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd" + SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1" BUILD: "true" - COMPILER: "gcc" @@ -204,7 +204,7 @@ - COMPILER: "clang" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd" + SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1" - COMPILER: "visual" HOST: "visual" diff --git a/programs/Makefile b/programs/Makefile index b6f3a916..e66d182f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -233,7 +233,7 @@ $(BUILD_DIR)/zstd : $(ZSTD_OBJ) @echo "$(LZMA_MSG)" @echo "$(LZ4_MSG)" @echo LINK $@ - $(LINK.o) $^ $(LDLIBS) -o $@$(EXT) + $(CC) $(FLAGS) $^ $(LDLIBS) -o $@$(EXT) ifeq ($(HAVE_HASH),1) SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd 2> $(VOID) | $(HASH) | cut -f 1 -d " ")