Merge pull request #2368 from facebook/progressive_libzstd

faster rebuild of libzstd
dev
Yann Collet 2020-10-22 17:36:56 -07:00 committed by GitHub
commit f90424da2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 94 additions and 59 deletions

View File

@ -10,13 +10,13 @@
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
# When cross-compiling from linux to windows, you might
# need to specify this as "Windows." Fedora build fails
# without it.
# When cross-compiling from linux to windows,
# you might need to specify this as "Windows."
# Fedora build fails without it.
#
# Note: mingw-w64 build from linux to windows does not
# fail on other tested distros (ubuntu, debian) even
# without manually specifying the TARGET_SYSTEM.
# Note: mingw-w64 build from linux to windows
# does not fail on other tested distros (ubuntu, debian)
# even without manually specifying the TARGET_SYSTEM.
TARGET_SYSTEM ?= $(OS)
# Version numbers
@ -35,28 +35,28 @@ CCVER := $(shell $(CC) --version)
# space-optimized defaults.
ZSTD_LIB_MINIFY ?= 0
ifneq ($(ZSTD_LIB_MINIFY), 0)
HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
ZSTD_LEGACY_SUPPORT ?= 0
ZSTD_LIB_DEPRECATED ?= 0
HUF_FORCE_DECOMPRESS_X1 ?= 1
ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
ZSTD_NO_INLINE ?= 1
ZSTD_STRIP_ERROR_STRINGS ?= 1
ifneq ($(HAVE_CC_OZ), 0)
# Some compilers (clang) support an even more space-optimized setting.
CFLAGS += -Oz
else
CFLAGS += -Os
endif
CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
-DDYNAMIC_BMI2=0 -DNDEBUG
HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
ZSTD_LEGACY_SUPPORT ?= 0
ZSTD_LIB_DEPRECATED ?= 0
HUF_FORCE_DECOMPRESS_X1 ?= 1
ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
ZSTD_NO_INLINE ?= 1
ZSTD_STRIP_ERROR_STRINGS ?= 1
ifneq ($(HAVE_CC_OZ), 0)
# Some compilers (clang) support an even more space-optimized setting.
CFLAGS += -Oz
else
CFLAGS += -O3
CFLAGS += -Os
endif
CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
-DDYNAMIC_BMI2=0 -DNDEBUG
else
CFLAGS += -O3
endif
CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@ -69,7 +69,7 @@ FLAGS = $(CPPFLAGS) $(CFLAGS)
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
GREP_OPTIONS += --color=never
GREP_OPTIONS += --color=never
endif
GREP = grep $(GREP_OPTIONS)
SED_ERE_OPT ?= -E
@ -82,7 +82,7 @@ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
ZSTD_FILES := $(ZSTDCOMMON_FILES)
ifeq ($(findstring GCC,$(CCVER)),GCC)
decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
endif
# Modules
@ -104,80 +104,90 @@ ZSTD_NO_INLINE ?= 0
ZSTD_STRIP_ERROR_STRINGS ?= 0
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
ZSTD_LIB_DICTBUILDER = 0
ZSTD_LIB_DEPRECATED = 0
ZSTD_LIB_DICTBUILDER = 0
ZSTD_LIB_DEPRECATED = 0
endif
ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
ZSTD_LEGACY_SUPPORT = 0
ZSTD_LIB_DEPRECATED = 0
ZSTD_LEGACY_SUPPORT = 0
ZSTD_LIB_DEPRECATED = 0
endif
ifneq ($(ZSTD_LIB_COMPRESSION), 0)
ZSTD_FILES += $(ZSTDCOMP_FILES)
ZSTD_FILES += $(ZSTDCOMP_FILES)
endif
ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
ZSTD_FILES += $(ZSTDDECOMP_FILES)
ZSTD_FILES += $(ZSTDDECOMP_FILES)
endif
ifneq ($(ZSTD_LIB_DEPRECATED), 0)
ZSTD_FILES += $(ZDEPR_FILES)
ZSTD_FILES += $(ZDEPR_FILES)
endif
ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
ZSTD_FILES += $(ZDICT_FILES)
ZSTD_FILES += $(ZDICT_FILES)
endif
ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
endif
ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
endif
ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
endif
ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
endif
ifneq ($(ZSTD_NO_INLINE), 0)
CFLAGS += -DZSTD_NO_INLINE
CFLAGS += -DZSTD_NO_INLINE
endif
ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
endif
ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
endif
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
endif
endif
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES))
ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
vpath %.c $(ZSTD_SUBDIR)
BUILD_DIR ?= obj
ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib
ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib
ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
# macOS linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
@ -191,7 +201,7 @@ lib-all: all
all: lib
libzstd.a: ARFLAGS = rcs
libzstd.a: $(ZSTD_OBJ)
libzstd.a: $(ZSTD_STATLIB_OBJ)
@echo compiling static library
$(Q)$(AR) $(ARFLAGS) $@ $^
@ -205,8 +215,9 @@ $(LIBZSTD): $(ZSTD_FILES)
else
LIBZSTD = libzstd.$(SHARED_EXT_VER)
$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
$(LIBZSTD): $(ZSTD_FILES)
$(LIBZSTD): CFLAGS += -fPIC
$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
$(LIBZSTD): $(ZSTD_DYNLIB_OBJ)
@echo compiling dynamic library $(LIBVER)
$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
@ -221,6 +232,7 @@ libzstd : $(LIBZSTD)
.PHONY: lib
lib : libzstd.a libzstd
# note : do not define lib-mt or lib-release as .PHONY
# make does not consider implicit pattern rule for .PHONY target
@ -234,6 +246,28 @@ lib : libzstd.a libzstd
@echo release build completed
# Generate .h dependencies automatically
DEPFLAGS = -MT $@ -MMD -MP -MF
$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
@echo $@
$(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
@echo $@
$(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
MKDIR ?= mkdir
$(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
$(Q)$(MKDIR) -p $@
DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
$(DEPFILES):
include $(wildcard $(DEPFILES))
# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
ZSTDMT_FILES = compress/zstdmt_compress.c
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
@ -248,6 +282,7 @@ clean:
$(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
$(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
$(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
$(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/*
@echo Cleaning library completed
#-----------------------------------------------------------------------------
@ -277,7 +312,7 @@ ifeq (,$(PCLIBDIR))
# Additional prefix check is required, since the empty string is technically a
# valid PCLIBDIR
ifeq (,$(shell echo "$(LIBDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(EXEC_PREFIX)(/|$$)@ p"))
$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file)
$(error configured libdir ($(LIBDIR)) is outside of prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
endif
endif
@ -285,20 +320,20 @@ ifeq (,$(PCINCDIR))
# Additional prefix check is required, since the empty string is technically a
# valid PCINCDIR
ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(PREFIX)(/|$$)@ p"))
$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFIX)), can't generate pkg-config file)
endif
endif
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
endif
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
INSTALL ?= ginstall
else
INSTALL ?= install
INSTALL ?= install
endif
INSTALL_PROGRAM ?= $(INSTALL)