From 2868d565e4b6efbcb48798b342b4803cb91376e0 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 13:18:11 +0200 Subject: [PATCH 01/74] test-related files moved to tests/ --- programs/Makefile | 136 +------------- tests/Makefile | 262 +++++++++++++++++++++++++-- {programs => tests}/datagencli.c | 0 {programs => tests}/fullbench.c | 0 {programs => tests}/fuzzer.c | 0 {programs => tests}/paramgrill.c | 0 {programs => tests}/playTests.sh | 0 {programs => tests}/roundTripCrash.c | 0 {programs => tests}/zbufftest.c | 0 {programs => tests}/zstreamtest.c | 0 10 files changed, 255 insertions(+), 143 deletions(-) rename {programs => tests}/datagencli.c (100%) rename {programs => tests}/fullbench.c (100%) rename {programs => tests}/fuzzer.c (100%) rename {programs => tests}/paramgrill.c (100%) rename {programs => tests}/playTests.sh (100%) mode change 100755 => 100644 rename {programs => tests}/roundTripCrash.c (100%) rename {programs => tests}/zbufftest.c (100%) rename {programs => tests}/zstreamtest.c (100%) diff --git a/programs/Makefile b/programs/Makefile index 9c70f868..b9d04edc 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -22,13 +22,11 @@ # - zstd homepage : http://www.zstd.net/ # ########################################################################## # zstd : Command Line Utility, supporting gzip-like arguments -# datagen : Synthetic and parametrable data generator, for tests -# fuzzer : Test tool, to check zstd integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode -# zbufftest : Test tool, to check ZBUFF integrity on target platform -# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode -# fullbench : Precisely measure speed for each zstd inner function -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# zstd32 : Same as zstd, but forced to compile in 32-bits mode +# zstd_nolegacy : zstd without support of decompression of legacy versions +# zstd-small: minimal zstd without dictBuilder and bench +# zstd-compress: compressor-only version of zstd +# zstd-decompress: decompressor-only version of zstd # ########################################################################## DESTDIR?= @@ -56,7 +54,6 @@ ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o @@ -80,17 +77,12 @@ EXT = VOID = /dev/null endif -ZBUFFTEST = -T2mn -FUZZERTEST= -T5mn -ZSTDRTTEST= --test-large-data -.PHONY: default all all32 clean install uninstall test test32 test-all +.PHONY: default all clean install uninstall default: zstd -all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen - -all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32 +all: zstd $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c @@ -136,58 +128,18 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ zstd-small: clean CFLAGS="-Os -s" $(MAKE) zstd-frugal -fullbench : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fullbench32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder -fuzzer : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder -fuzzer32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -zbufftest : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -zbufftest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -zstreamtest : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c zstreamtest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -zstreamtest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) datagen.c zstreamtest.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) - -paramgrill : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c paramgrill.c - $(CC) $(FLAGS) $^ -lm -o $@$(EXT) - -datagen : datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -roundTripCrash : $(ZSTDDECOMP_O) $(ZSTD_FILES) roundTripCrash.c - $(CC) $(FLAGS) $^ -o $@$(EXT) clean: $(MAKE) -C ../lib clean @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ - zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ - fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) + zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) @echo Cleaning completed -cleano32: - @rm -f ../lib/decompress/*.o #---------------------------------------------------------------------------------- #make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets #---------------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly)) -HOST_OS = POSIX install: zstd @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @@ -208,76 +160,4 @@ uninstall: rm -f $(DESTDIR)$(MANDIR)/unzstd.1 [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: zstd datagen fuzzer fullbench zbufftest - @echo "\n ---- valgrind tests : memory analyzer ----" - $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) - @rm tmp - $(VALGRIND) ./fuzzer -T1mn -t1 - $(VALGRIND) ./fullbench -i1 - $(VALGRIND) ./zbufftest -T1mn - -endif - - -ifneq (,$(filter MSYS%,$(shell uname))) -HOST_OS = MSYS -endif - - -#------------------------------------------------------------------------ -#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets -#------------------------------------------------------------------------ -ifneq (,$(filter $(HOST_OS),MSYS POSIX)) -zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) - -test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream - -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 - -test-all: test test32 valgrindTest - -test-zstd: ZSTD = ./zstd -test-zstd: zstd zstd-playTests - -test-zstd32: ZSTD = ./zstd32 -test-zstd32: zstd32 zstd-playTests - -test-zstd_nolegacy: ZSTD = ./zstd -test-zstd_nolegacy: zstd_nolegacy zstd-playTests - -test-fullbench: fullbench datagen - ./fullbench -i1 - ./fullbench -i1 -P0 - -test-fullbench32: fullbench32 datagen - ./fullbench32 -i1 - ./fullbench32 -i1 -P0 - -test-fuzzer: fuzzer - ./fuzzer $(FUZZERTEST) - -test-fuzzer32: fuzzer32 - ./fuzzer32 $(FUZZERTEST) - -test-zbuff: zbufftest - ./zbufftest $(ZBUFFTEST) - -test-zbuff32: zbufftest32 - ./zbufftest32 $(ZBUFFTEST) - -test-zstream: zstreamtest - ./zstreamtest $(ZBUFFTEST) - -test-zstream32: zstreamtest32 - ./zstreamtest32 $(ZBUFFTEST) - endif diff --git a/tests/Makefile b/tests/Makefile index d83a1dfc..33db59e1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,6 @@ # ########################################################################## -# Zstd tests - Makefile +# ZSTD programs - Makefile # Copyright (C) Yann Collet 2015-2016 -# All rights reserved. # # GPL v2 License # @@ -20,27 +19,260 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # You can contact the author at : -# - ZSTD homepage : http://www.zstd.net/ +# - zstd homepage : http://www.zstd.net/ # ########################################################################## -# versionstest : Compatibility test between zstd versions stored on Github (v0.1+) +# zstd : Command Line Utility, supporting gzip-like arguments +# datagen : Synthetic and parametrable data generator, for tests +# fuzzer : Test tool, to check zstd integrity on target platform +# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# zbufftest : Test tool, to check ZBUFF integrity on target platform +# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode +# fullbench : Precisely measure speed for each zstd inner function +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -PYTHON ?= python3 -TESTARTEFACT := versionsTest namespaceTest +DESTDIR?= +PREFIX ?= /usr/local +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 -.PHONY: default all clean namespaceTest versionsTest +ZSTDDIR = ../lib -default: all +ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1) +ALIGN_LOOP = -falign-loops=32 +else +ALIGN_LOOP = +endif -all: namespaceTest versionsTest +CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -namespaceTest: - if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi - $(RM) $@ -versionsTest: - $(PYTHON) test-zstd-versions.py +ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c +ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c +ZSTDDECOMP_O = + + +ifeq ($(ZSTD_LEGACY_SUPPORT), 0) +CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 +ZSTDLEGACY_FILES:= +else +ZSTD_LEGACY_SUPPORT:=1 +CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy +ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c +endif + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +VOID = nul +else +EXT = +VOID = /dev/null +endif + +ZBUFFTEST = -T2mn +FUZZERTEST= -T5mn +ZSTDRTTEST= --test-large-data + +.PHONY: default all all32 clean install uninstall test test32 test-all + +default: zstd + +all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen + +all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32 + + + +zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ + zstdcli.c fileio.c bench.c datagen.c dibio.c + $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + +zstd32 : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ + zstdcli.c fileio.c bench.c datagen.c dibio.c + $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + + +zstd_nolegacy : + $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 + +zstd-pgo : MOREFLAGS = -fprofile-generate +zstd-pgo : clean zstd + ./zstd -b19i1 $(PROFILE_WITH) + ./zstd -b16i1 $(PROFILE_WITH) + ./zstd -b9i2 $(PROFILE_WITH) + ./zstd -b $(PROFILE_WITH) + ./zstd -b7i2 $(PROFILE_WITH) + ./zstd -b5 $(PROFILE_WITH) + rm zstd + $(MAKE) zstd MOREFLAGS=-fprofile-use + +zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) + +zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ + zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) + +zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ + zstdcli.c fileio.c + $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) + +zstd-small: clean + CFLAGS="-Os -s" $(MAKE) zstd-frugal + +fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + +fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder +fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + +zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + +zstreamtest : $(ZSTD_FILES) datagen.c zstreamtest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zstreamtest32 : $(ZSTD_FILES) datagen.c zstreamtest.c + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + +paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c + $(CC) $(FLAGS) $^ -lm -o $@$(EXT) + +datagen : datagen.c datagencli.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +roundTripCrash : $(ZSTD_FILES) roundTripCrash.c + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: - @$(RM) -fR $(TESTARTEFACT) + $(MAKE) -C ../lib clean + @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ + zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ + fullbench$(EXT) fullbench32$(EXT) \ + fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) @echo Cleaning completed + +cleano32: + @rm -f ../lib/decompress/*.o + +#---------------------------------------------------------------------------------- +#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets +#---------------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly)) +HOST_OS = POSIX +install: zstd + @echo Installing binaries + @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat + @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd + @echo Installing man pages + @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 + @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1 + @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1 + @echo zstd installation completed + +uninstall: + rm -f $(DESTDIR)$(BINDIR)/zstdcat + rm -f $(DESTDIR)$(BINDIR)/unzstd + [ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT) + rm -f $(DESTDIR)$(MANDIR)/zstdcat.1 + rm -f $(DESTDIR)$(MANDIR)/unzstd.1 + [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 + @echo zstd programs successfully uninstalled + +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench zbufftest + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp + $(VALGRIND) ./zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + $(VALGRIND) ./zbufftest -T1mn + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +zstd-playTests: datagen + ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) + +test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream + +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 + +test-all: test test32 valgrindTest + +test-zstd: ZSTD = ./zstd +test-zstd: zstd zstd-playTests + +test-zstd32: ZSTD = ./zstd32 +test-zstd32: zstd32 zstd-playTests + +test-zstd_nolegacy: ZSTD = ./zstd +test-zstd_nolegacy: zstd_nolegacy zstd-playTests + +test-fullbench: fullbench datagen + ./fullbench -i1 + ./fullbench -i1 -P0 + +test-fullbench32: fullbench32 datagen + ./fullbench32 -i1 + ./fullbench32 -i1 -P0 + +test-fuzzer: fuzzer + ./fuzzer $(FUZZERTEST) + +test-fuzzer32: fuzzer32 + ./fuzzer32 $(FUZZERTEST) + +test-zbuff: zbufftest + ./zbufftest $(ZBUFFTEST) + +test-zbuff32: zbufftest32 + ./zbufftest32 $(ZBUFFTEST) + +test-zstream: zstreamtest + ./zstreamtest $(ZBUFFTEST) + +test-zstream32: zstreamtest32 + ./zstreamtest32 $(ZBUFFTEST) + +endif diff --git a/programs/datagencli.c b/tests/datagencli.c similarity index 100% rename from programs/datagencli.c rename to tests/datagencli.c diff --git a/programs/fullbench.c b/tests/fullbench.c similarity index 100% rename from programs/fullbench.c rename to tests/fullbench.c diff --git a/programs/fuzzer.c b/tests/fuzzer.c similarity index 100% rename from programs/fuzzer.c rename to tests/fuzzer.c diff --git a/programs/paramgrill.c b/tests/paramgrill.c similarity index 100% rename from programs/paramgrill.c rename to tests/paramgrill.c diff --git a/programs/playTests.sh b/tests/playTests.sh old mode 100755 new mode 100644 similarity index 100% rename from programs/playTests.sh rename to tests/playTests.sh diff --git a/programs/roundTripCrash.c b/tests/roundTripCrash.c similarity index 100% rename from programs/roundTripCrash.c rename to tests/roundTripCrash.c diff --git a/programs/zbufftest.c b/tests/zbufftest.c similarity index 100% rename from programs/zbufftest.c rename to tests/zbufftest.c diff --git a/programs/zstreamtest.c b/tests/zstreamtest.c similarity index 100% rename from programs/zstreamtest.c rename to tests/zstreamtest.c From a1c8115df59f1bb7e01b4529fca07f7a89cb3e13 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 13:35:36 +0200 Subject: [PATCH 02/74] improved tests\Makefile --- Makefile | 8 ++--- tests/Makefile | 81 +++++--------------------------------------------- 2 files changed, 11 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index 32f063f0..0d5b1cc2 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ PRGDIR = programs ZSTDDIR = lib ZWRAPDIR = zlibWrapper +TESTDIR = tests # Define nul output ifneq (,$(filter Windows%,$(OS))) @@ -47,9 +48,7 @@ default: zstd all: $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ - @rm -f lib/decompress/*.o - $(MAKE) -C $(PRGDIR) all32 + $(MAKE) -C $(PRGDIR) $@ zstd32 zstd: $(MAKE) -C $(PRGDIR) @@ -60,11 +59,12 @@ zlibwrapper: $(MAKE) -C $(ZWRAPDIR) all test: - $(MAKE) -C $(PRGDIR) $@ + $(MAKE) -C $(TESTDIR) $@ clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(TESTDIR) $@ > $(VOID) @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) @rm -f zstd @echo Cleaning completed diff --git a/tests/Makefile b/tests/Makefile index 33db59e1..bbe6f183 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -35,14 +35,8 @@ DESTDIR?= PREFIX ?= /usr/local BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 - ZSTDDIR = ../lib -ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1) -ALIGN_LOOP = -falign-loops=32 -else -ALIGN_LOOP = -endif CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder CFLAGS ?= -O3 @@ -58,18 +52,8 @@ ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompres ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c -ZSTDDECOMP_O = -ifeq ($(ZSTD_LEGACY_SUPPORT), 0) -CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 -ZSTDLEGACY_FILES:= -else -ZSTD_LEGACY_SUPPORT:=1 -CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy -ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c -endif - # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -90,46 +74,17 @@ default: zstd all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen -all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32 +all32: zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32 -zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ +zstd : $(ZSTD_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c - $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) + $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) -zstd32 : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ +zstd32 : $(ZSTD_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c - $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) - - -zstd_nolegacy : - $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 - -zstd-pgo : MOREFLAGS = -fprofile-generate -zstd-pgo : clean zstd - ./zstd -b19i1 $(PROFILE_WITH) - ./zstd -b16i1 $(PROFILE_WITH) - ./zstd -b9i2 $(PROFILE_WITH) - ./zstd -b $(PROFILE_WITH) - ./zstd -b7i2 $(PROFILE_WITH) - ./zstd -b5 $(PROFILE_WITH) - rm zstd - $(MAKE) zstd MOREFLAGS=-fprofile-use - -zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) - -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) - -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ - zstdcli.c fileio.c - $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) - -zstd-small: clean - CFLAGS="-Os -s" $(MAKE) zstd-frugal + $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -169,40 +124,18 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c clean: $(MAKE) -C ../lib clean @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ - zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ + zstd$(EXT) zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) @echo Cleaning completed -cleano32: - @rm -f ../lib/decompress/*.o #---------------------------------------------------------------------------------- -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets +#make valgrindTest is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets #---------------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly)) HOST_OS = POSIX -install: zstd - @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd - @echo Installing man pages - @install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1 - @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1 - @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1 - @echo zstd installation completed - -uninstall: - rm -f $(DESTDIR)$(BINDIR)/zstdcat - rm -f $(DESTDIR)$(BINDIR)/unzstd - [ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT) - rm -f $(DESTDIR)$(MANDIR)/zstdcat.1 - rm -f $(DESTDIR)$(MANDIR)/unzstd.1 - [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 - @echo zstd programs successfully uninstalled valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 valgrindTest: zstd datagen fuzzer fullbench zbufftest From 2d5eb2beb967af322f629fb4edb9b90dfd08c577 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 13:53:26 +0200 Subject: [PATCH 03/74] updated .travis.yml and appveyor.yml for tests/ --- .travis.yml | 6 ++--- Makefile | 21 ++++++++-------- appveyor.yml | 2 +- tests/Makefile | 68 +++++++++++++++++++++++++------------------------- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42f0043b..350fc1e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest" + env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest" - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" @@ -39,7 +39,7 @@ matrix: - qemu-user-static - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C programs test32" + env: PLATFORM="Ubuntu 12.04" CMD="make -C tests test32" addons: apt: packages: @@ -58,7 +58,7 @@ matrix: - gcc-multilib - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C programs valgrindTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C tests valgrindTest" addons: apt: packages: diff --git a/Makefile b/Makefile index 0d5b1cc2..a6b2fb42 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ default: zstd all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ zstd32 + $(MAKE) -C $(TESTDIR) $@ all32 zstd: $(MAKE) -C $(PRGDIR) @@ -102,16 +103,16 @@ clangtest: clean $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" armtest: clean - $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" ppctest: clean - $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" ppc64test: clean - $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -static" + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -static" usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" @@ -123,7 +124,7 @@ msan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason asan32: clean - $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" @@ -163,13 +164,13 @@ c11test: clean CFLAGS="-std=c11" $(MAKE) all bmix64test: clean - CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test bmix32test: clean - CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test bmi32test: clean - CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test staticAnalyze: clean CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all diff --git a/appveyor.yml b/appveyor.yml index 03fb7930..4b4ef6c6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,7 +24,7 @@ install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%COMPILER%]==[gcc] ( - SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && + SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe && diff --git a/tests/Makefile b/tests/Makefile index bbe6f183..e0689ce6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,5 @@ # ########################################################################## -# ZSTD programs - Makefile +# ZSTD tests - Makefile # Copyright (C) Yann Collet 2015-2016 # # GPL v2 License @@ -21,7 +21,6 @@ # You can contact the author at : # - zstd homepage : http://www.zstd.net/ # ########################################################################## -# zstd : Command Line Utility, supporting gzip-like arguments # datagen : Synthetic and parametrable data generator, for tests # fuzzer : Test tool, to check zstd integrity on target platform # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode @@ -36,9 +35,9 @@ PREFIX ?= /usr/local BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib +PRGDIR = ../programs - -CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder +CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR) CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef @@ -68,54 +67,55 @@ ZBUFFTEST = -T2mn FUZZERTEST= -T5mn ZSTDRTTEST= --test-large-data -.PHONY: default all all32 clean install uninstall test test32 test-all +.PHONY: default all all32 clean test test32 test-all -default: zstd +default: fullbench -all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen +all: fullbench fuzzer zbufftest zstreamtest paramgrill datagen -all32: zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32 +all32: fullbench32 fuzzer32 zbufftest32 zstreamtest32 -zstd : $(ZSTD_FILES) $(ZDICT_FILES) \ - zstdcli.c fileio.c bench.c datagen.c dibio.c - $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) +zstd: + $(MAKE) -C $(PRGDIR) $@ -zstd32 : $(ZSTD_FILES) $(ZDICT_FILES) \ - zstdcli.c fileio.c bench.c datagen.c dibio.c - $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) +zstd32: + $(MAKE) -C $(PRGDIR) $@ -fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c +zstd_nolegacy: + $(MAKE) -C $(PRGDIR) $@ + +fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c +fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder -fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c +fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder -fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c +fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c +zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c +zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -zstreamtest : $(ZSTD_FILES) datagen.c zstreamtest.c +zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zstreamtest32 : $(ZSTD_FILES) datagen.c zstreamtest.c +zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c +paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) -datagen : datagen.c datagencli.c +datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) roundTripCrash : $(ZSTD_FILES) roundTripCrash.c @@ -124,7 +124,7 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c clean: $(MAKE) -C ../lib clean @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ - zstd$(EXT) zstd32$(EXT) \ + $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) @@ -141,12 +141,12 @@ valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 valgrindTest: zstd datagen fuzzer fullbench zbufftest @echo "\n ---- valgrind tests : memory analyzer ----" $(VALGRIND) ./datagen -g50M > $(VOID) - $(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi - ./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID) - ./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID) - ./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp - $(VALGRIND) ./zstd -vdf tmp -o $(VOID) - ./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID) + $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID) + ./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp + $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -o $(VOID) + ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID) @rm tmp $(VALGRIND) ./fuzzer -T1mn -t1 $(VALGRIND) ./fullbench -i1 @@ -173,13 +173,13 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 test-all: test test32 valgrindTest -test-zstd: ZSTD = ./zstd +test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd zstd-playTests -test-zstd32: ZSTD = ./zstd32 +test-zstd32: ZSTD = $(PRGDIR)/zstd32 test-zstd32: zstd32 zstd-playTests -test-zstd_nolegacy: ZSTD = ./zstd +test-zstd_nolegacy: ZSTD = $(PRGDIR)/zstd test-zstd_nolegacy: zstd_nolegacy zstd-playTests test-fullbench: fullbench datagen From 63e335704efac0977f8b45fe56fdbaeb903d338a Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 14:04:57 +0200 Subject: [PATCH 04/74] updated scripts +x permissions --- tests/Makefile | 16 ++++++++++++++-- tests/playTests.sh | 0 2 files changed, 14 insertions(+), 2 deletions(-) mode change 100644 => 100755 tests/playTests.sh diff --git a/tests/Makefile b/tests/Makefile index e0689ce6..a8fb97c2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,6 +28,7 @@ # zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode # fullbench : Precisely measure speed for each zstd inner function # fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# versionstest : Compatibility test between zstd versions stored on Github (v0.1+) # ########################################################################## DESTDIR?= @@ -36,6 +37,9 @@ BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib PRGDIR = ../programs +PYTHON ?= python3 +TESTARTEFACT := versionsTest namespaceTest + CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR) CFLAGS ?= -O3 @@ -67,7 +71,7 @@ ZBUFFTEST = -T2mn FUZZERTEST= -T5mn ZSTDRTTEST= --test-large-data -.PHONY: default all all32 clean test test32 test-all +.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest default: fullbench @@ -121,9 +125,17 @@ datagen : $(PRGDIR)/datagen.c datagencli.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) +namespaceTest: + if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi + $(RM) $@ + +versionsTest: + $(PYTHON) test-zstd-versions.py + clean: $(MAKE) -C ../lib clean - @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ + @$(RM) -fR $(TESTARTEFACT) + @$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ diff --git a/tests/playTests.sh b/tests/playTests.sh old mode 100644 new mode 100755 From b1421f2d366e0c0b2a390bbff8b85452afc7dfa8 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 14:17:11 +0200 Subject: [PATCH 05/74] updated VS projects --- projects/VS2005/fullbench/fullbench.vcproj | 2 +- projects/VS2005/fuzzer/fuzzer.vcproj | 2 +- projects/VS2008/fullbench/fullbench.vcproj | 2 +- projects/VS2008/fuzzer/fuzzer.vcproj | 2 +- projects/VS2010/datagen/datagen.vcxproj | 4 ++-- projects/VS2010/fullbench/fullbench.vcxproj | 4 ++-- projects/VS2010/fuzzer/fuzzer.vcxproj | 4 ++-- projects/VS2010/zstdlib/zstdlib.vcxproj | 3 +-- 8 files changed, 11 insertions(+), 12 deletions(-) diff --git a/projects/VS2005/fullbench/fullbench.vcproj b/projects/VS2005/fullbench/fullbench.vcproj index 6379cf26..8aab0820 100644 --- a/projects/VS2005/fullbench/fullbench.vcproj +++ b/projects/VS2005/fullbench/fullbench.vcproj @@ -348,7 +348,7 @@ > + @@ -157,7 +157,7 @@ - + diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj index 159a58d5..882c78f8 100644 --- a/projects/VS2010/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -1,4 +1,4 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/projects/VS2010/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj index 5c8d800b..34b61053 100644 --- a/projects/VS2010/fuzzer/fuzzer.vcxproj +++ b/projects/VS2010/fuzzer/fuzzer.vcxproj @@ -1,4 +1,4 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index 8a5bc8b1..232fdf44 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -1,4 +1,4 @@ - + @@ -43,7 +43,6 @@ - From 49775e3a60648932573408198f07afa70349e037 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 14:25:28 +0200 Subject: [PATCH 06/74] VS projects: fixed include directiories --- projects/VS2005/fullbench/fullbench.vcproj | 8 ++++---- projects/VS2005/fuzzer/fuzzer.vcproj | 8 ++++---- projects/VS2008/fullbench/fullbench.vcproj | 8 ++++---- projects/VS2008/fuzzer/fuzzer.vcproj | 8 ++++---- projects/VS2010/datagen/datagen.vcxproj | 8 ++++---- projects/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- projects/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/projects/VS2005/fullbench/fullbench.vcproj b/projects/VS2005/fullbench/fullbench.vcproj index 8aab0820..17af9c81 100644 --- a/projects/VS2005/fullbench/fullbench.vcproj +++ b/projects/VS2005/fullbench/fullbench.vcproj @@ -43,7 +43,7 @@ true false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ true false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ false false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj index 882c78f8..d342bcb6 100644 --- a/projects/VS2010/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -65,24 +65,24 @@ true - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false $(Platform)\$(Configuration)\ true - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false $(Platform)\$(Configuration)\ false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); false diff --git a/projects/VS2010/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj index 34b61053..a436b038 100644 --- a/projects/VS2010/fuzzer/fuzzer.vcxproj +++ b/projects/VS2010/fuzzer/fuzzer.vcxproj @@ -66,24 +66,24 @@ true false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ true false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); $(Platform)\$(Configuration)\ false false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); From 6ca9b853a1ed858ed3c15f915ca39d2b3a9f3636 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 14:42:28 +0200 Subject: [PATCH 07/74] cmake project: fixed include directories --- projects/cmake/CMakeLists.txt | 1 + projects/cmake/programs/.gitignore | 5 --- projects/cmake/programs/CMakeLists.txt | 15 ------- projects/cmake/tests/.gitignore | 7 +++ projects/cmake/tests/CMakeLists.txt | 61 ++++++++++++++++++++++++++ tests/playTests.sh | 9 ++-- 6 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 projects/cmake/tests/.gitignore create mode 100644 projects/cmake/tests/CMakeLists.txt diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index 9650eb4f..86178bf9 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -46,6 +46,7 @@ ENDIF (ZSTD_LEGACY_SUPPORT) ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) +ADD_SUBDIRECTORY(tests) #----------------------------------------------------------------------------- # Add extra compilation flags diff --git a/projects/cmake/programs/.gitignore b/projects/cmake/programs/.gitignore index 81eec349..f04c5b42 100644 --- a/projects/cmake/programs/.gitignore +++ b/projects/cmake/programs/.gitignore @@ -1,8 +1,3 @@ # produced by make -datagen -fullbench -fuzzer -paramgrill -zbufftest zstd zstd-frugal diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index fddfc7df..163e944c 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -51,23 +51,8 @@ ENDIF (ZSTD_LEGACY_SUPPORT) ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY}) TARGET_LINK_LIBRARIES(zstd libzstd_static) -ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c) -TARGET_LINK_LIBRARIES(fullbench libzstd_static) - -ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c) -TARGET_LINK_LIBRARIES(fuzzer libzstd_static) - IF (UNIX) ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static) SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") - - ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c) - TARGET_LINK_LIBRARIES(zbufftest libzstd_static) - - ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c) - TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library - - ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c) - TARGET_LINK_LIBRARIES(datagen libzstd_static) ENDIF (UNIX) diff --git a/projects/cmake/tests/.gitignore b/projects/cmake/tests/.gitignore new file mode 100644 index 00000000..2ab62a3e --- /dev/null +++ b/projects/cmake/tests/.gitignore @@ -0,0 +1,7 @@ +# produced by make +datagen +fullbench +fuzzer +paramgrill +zbufftest + diff --git a/projects/cmake/tests/CMakeLists.txt b/projects/cmake/tests/CMakeLists.txt new file mode 100644 index 00000000..f5ece894 --- /dev/null +++ b/projects/cmake/tests/CMakeLists.txt @@ -0,0 +1,61 @@ +# ################################################################ +# zstd - Makefile +# Copyright (C) Yann Collet 2014-2016 +# All rights reserved. +# +# BSD license +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, this +# list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ################################################################ + +PROJECT(tests) + +SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +# Define project root directory +SET(ROOT_DIR ../../..) + +# Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) +SET(PROGRAMS_DIR ${ROOT_DIR}/programs) +SET(TESTS_DIR ${ROOT_DIR}/tests) +INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) + +ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c) +TARGET_LINK_LIBRARIES(fullbench libzstd_static) + +ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c) +TARGET_LINK_LIBRARIES(fuzzer libzstd_static) + +IF (UNIX) + ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c) + TARGET_LINK_LIBRARIES(zbufftest libzstd_static) + + ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c) + TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library + + ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c) + TARGET_LINK_LIBRARIES(datagen libzstd_static) +ENDIF (UNIX) diff --git a/tests/playTests.sh b/tests/playTests.sh index 70ac35ec..b454a6db 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -175,12 +175,12 @@ $ECHO "\n**** dictionary tests **** " diff -q tmp1 tmp2 $ECHO "- Create first dictionary" $ZSTD --train *.c -o tmpDict -cp zstdcli.c tmp +cp fuzzer.c tmp $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -of result -diff zstdcli.c result +diff fuzzer.c result $ECHO "- Create second (different) dictionary" -$ZSTD --train *.c *.h -o tmpDictC +$ZSTD --train *.c -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" $ZSTD --train *.c --dictID 1 -o tmpDict1 @@ -188,12 +188,11 @@ cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Compress without dictID" $ZSTD -f tmp -D tmpDict1 --no-dictID $ZSTD -d tmp.zst -D tmpDict -of result -diff zstdcli.c result +diff fuzzer.c result $ECHO "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict cp *.c dirTestDict -cp *.h dirTestDict cat dirTestDict/* | $MD5SUM > tmph1 # note : we expect same file order to generate same hash $ZSTD -f dirTestDict/* -D tmpDictC $ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2 From b7d3449d5dfd768b9934fa429ef645f8ff567265 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 15:13:41 +0200 Subject: [PATCH 08/74] playTests.sh working with tests/ --- tests/playTests.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index b454a6db..013ba6f6 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -169,30 +169,33 @@ $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" $ECHO "\n**** dictionary tests **** " +TESTFILE=../programs/zstdcli.c ./datagen > tmpDict ./datagen -g1M | $MD5SUM > tmp1 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 diff -q tmp1 tmp2 $ECHO "- Create first dictionary" -$ZSTD --train *.c -o tmpDict -cp fuzzer.c tmp +$ZSTD --train *.c ../programs/*.c -o tmpDict +cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -of result -diff fuzzer.c result +diff $TESTFILE result $ECHO "- Create second (different) dictionary" -$ZSTD --train *.c -o tmpDictC +$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" -$ZSTD --train *.c --dictID 1 -o tmpDict1 +$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Compress without dictID" $ZSTD -f tmp -D tmpDict1 --no-dictID $ZSTD -d tmp.zst -D tmpDict -of result -diff fuzzer.c result +diff $TESTFILE result $ECHO "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict cp *.c dirTestDict +cp ../programs/*.c dirTestDict +cp ../programs/*.h dirTestDict cat dirTestDict/* | $MD5SUM > tmph1 # note : we expect same file order to generate same hash $ZSTD -f dirTestDict/* -D tmpDictC $ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2 From 7feafdd7b80868762805db8ff3fdff75ca3dba72 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 15:17:18 +0200 Subject: [PATCH 09/74] appveyor.yml working with tests/ --- appveyor.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4b4ef6c6..4e21db1c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,7 +44,7 @@ build_script: ECHO *** && ECHO make %CLANG_PARAMS% && make %CLANG_PARAMS% && - COPY programs\fuzzer.exe projects\fuzzer_clang.exe && + COPY tests\fuzzer.exe tests\fuzzer_clang.exe && make clean ) - if [%COMPILER%]==[gcc] ( @@ -64,7 +64,7 @@ build_script: msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -74,7 +74,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -84,7 +84,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -94,7 +94,7 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && @@ -104,26 +104,25 @@ build_script: msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe && MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\ ) test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - SET FUZZERTEST=-T1mn - if [%COMPILER%]==[gcc] ( - if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST% + if [%PLATFORM%]==[mingw64] tests\fuzzer_clang.exe %FUZZERTEST% ) - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( - CD programs && + CD tests && SET ZSTD=./zstd.exe && sh -e playTests.sh --test-large-data && fullbench.exe -i1 && fullbench.exe -i1 -P0 && - CD .. && - projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && - projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% + fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && + fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && + fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && + fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% && + fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) From b6514d24966dfd0a675e14c11736f03d95ed4039 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 18 Aug 2016 16:53:29 +0200 Subject: [PATCH 10/74] improved .gitignore --- programs/.gitignore | 27 --------------------------- tests/.gitignore | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/programs/.gitignore b/programs/.gitignore index 369a27cc..5875fc79 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -3,42 +3,16 @@ zstd zstd32 zstd-compress zstd-decompress -fullbench -fullbench32 -fuzzer -fuzzer32 -zbufftest -zbufftest32 -zstreamtest -zstreamtest32 -datagen -paramgrill -paramgrill32 -roundTripCrash # Object files *.o *.ko -# Libraries -*.lib -*.a - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - # Executables *.exe *.out *.app -# Visual solution files -*.suo -*.user - # Default result files dictionary grillResults.txt @@ -53,5 +27,4 @@ afl # Misc files *.bat -fileTests.sh dirTest* diff --git a/tests/.gitignore b/tests/.gitignore index 5ea6b5b7..c8f9ae79 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,17 @@ +# local binary (Makefile) +fullbench +fullbench32 +fuzzer +fuzzer32 +zbufftest +zbufftest32 +zstreamtest +zstreamtest32 +datagen +paramgrill +paramgrill32 +roundTripCrash + # Tmp test directory zstdtest speedTest @@ -7,3 +21,28 @@ namespaceTest # Local script startSpeedTest speedTest.pid + +# Object files +*.o +*.ko + +# Executables +*.exe +*.out +*.app + +# Default result files +dictionary +grillResults.txt +_* +tmp* +*.zst +result +out + +# fuzzer +afl + +# Misc files +*.bat +dirTest* From c629776b31bcea89558f69760df10e8bfdd4b4dd Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 19 Aug 2016 10:45:59 +0200 Subject: [PATCH 11/74] make clean will remove decompress/zstd_decompress.o --- programs/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/Makefile b/programs/Makefile index b9d04edc..d738f40c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -131,6 +131,7 @@ zstd-small: clean clean: $(MAKE) -C ../lib clean + @rm -f ../lib/decompress/*.o @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) @echo Cleaning completed From da3fbcb3023369690e31c9691f1580a491b704c3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 19 Aug 2016 14:23:58 +0200 Subject: [PATCH 12/74] Added ZDICT_getDictID() --- lib/dictBuilder/zdict.c | 7 +++++++ lib/dictBuilder/zdict.h | 1 + tests/fuzzer.c | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 0e44c425..b2fb7c08 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -126,6 +126,13 @@ unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + /*-******************************************************** * Dictionary training functions diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 67dba70d..68349da0 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -68,6 +68,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap /*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index be36d74b..87647b9d 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -272,6 +272,12 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZDICT_isError(dictSize)) goto _output_error; DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize); + DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++); + { U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize); + if (dictID==0) goto _output_error; + DISPLAYLEVEL(4, "OK : %u \n", dictID); + } + DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), CNBuffer, CNBuffSize, From fdba57d513732e36dfee83bb55026dd0565485c1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 19 Aug 2016 18:32:30 +0200 Subject: [PATCH 13/74] update version number --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index dbca06cd..24dde4da 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -55,7 +55,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 8 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 2 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 7be46bf8f2cda11f9e6a43b2529bdfc99248ef7c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 19 Aug 2016 18:39:36 +0200 Subject: [PATCH 14/74] promoted streaming API to stable (except _advanced() variants) --- lib/zstd.h | 195 ++++++++++++++++++++++++++++------------------------- 1 file changed, 102 insertions(+), 93 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 24dde4da..d07cc3c7 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -191,6 +191,107 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); +/*-************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + +/*====== compression ======*/ + +/*-*********************************************************************** +* Streaming compression - howto +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* +* Start by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically update both `pos`. +* Note that it may not consume the entire input, in which case `pos < size`, +* and it's up to the caller to present again remaining data. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* `output->pos` will be updated. +* Note some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTD_CStream* ZSTD_createCStream(void); +size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*====== decompression ======*/ + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-init multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos`. +* Note that it may not consume the entire input (pos < size), +* in which case it's up to the caller to present remaining input again. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************************/ + +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTD_DStream* ZSTD_createDStream(void); +size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ + +size_t ZSTD_initDStream(ZSTD_DStream* zds); +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + + + #ifdef ZSTD_STATIC_LINKING_ONLY /* ==================================================================================== @@ -321,110 +422,18 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Streaming +* Advanced Streaming ********************************************************************/ -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - /*====== compression ======*/ -/*-*********************************************************************** -* Streaming compression - howto -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* -* Start by initializing ZSTD_CStream. -* Use ZSTD_initCStream() to start a new compression operation. -* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. -* -* Use ZSTD_compressStream() repetitively to consume input stream. -* The function will automatically update both `pos`. -* Note that it may not consume the entire input, in which case `pos < size`, -* and it's up to the caller to present again remaining data. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). -* `output->pos` will be updated. -* Note some content might still be left within internal buffer if `output->size` is too small. -* @return : nb of bytes still present within internal buffer (0 if it's empty) -* or an error code, which can be tested using ZSTD_isError(). -* -* ZSTD_endStream() instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. -* In which case, call again ZSTD_endStream() to complete the flush. -* @return : nb of bytes still present within internal buffer (0 if it's empty) -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef struct ZSTD_CStream_s ZSTD_CStream; -ZSTD_CStream* ZSTD_createCStream(void); -size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ - -size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); - -/* advanced */ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); - /*====== decompression ======*/ -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-init multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation, -* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos`. -* Note that it may not consume the entire input (pos < size), -* in which case it's up to the caller to present remaining input again. -* @return : 0 when a frame is completely decoded and fully flushed, -* 1 when there is still some data left within internal buffer to flush, -* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************************/ - -typedef struct ZSTD_DStream_s ZSTD_DStream; -ZSTD_DStream* ZSTD_createDStream(void); -size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ - -size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - /* advanced */ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); From af1960396ba8c801f706c4851a18944febe6c3cf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 19 Aug 2016 19:38:19 +0200 Subject: [PATCH 15/74] sync huff0 --- lib/decompress/huf_decompress.c | 57 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 1580b375..a5521bd3 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -37,7 +37,7 @@ ****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) /* inline is defined */ -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) || defined(__GNUC__) # define inline __inline #else # define inline /* disable inline */ @@ -57,10 +57,10 @@ /* ************************************************************** -* Includes +* Dependencies ****************************************************************/ #include /* memcpy, memset */ -#include "bitstream.h" +#include "bitstream.h" /* BIT_* */ #include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" @@ -103,7 +103,7 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; @@ -388,22 +388,22 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co } /* fill DTable */ - { U32 s; for (s=0; s= 1 */ + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i= 1 */ - rankVal[weight] += length; - }} + rankVal[weight] += length; + } } } typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1]; @@ -442,8 +442,8 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, MEM_writeLE16(&(DElt.sequence), symbol); DElt.nbBits = (BYTE)(nbBits); DElt.length = 1; - { U32 u; - const U32 end = start + length; + { U32 const end = start + length; + U32 u; for (u = start; u < end; u++) DTable[u] = DElt; } } rankVal[weight] += length; @@ -467,7 +467,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); - //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; @@ -533,7 +533,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) { - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ memcpy(op, dt+val, 2); BIT_skipBits(DStream, dt[val].nbBits); return dt[val].length; @@ -541,7 +541,7 @@ static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4 static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) { - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ memcpy(op, dt+val, 1); if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); else { @@ -570,7 +570,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c BYTE* const pStart = p; /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_1(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr); @@ -578,7 +578,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c } /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) HUF_DECODE_SYMBOLX4_0(p, bitDPtr); while (p <= pEnd-2) @@ -697,7 +697,7 @@ static size_t HUF_decompress4X4_usingDTable_internal( /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op3, &bitD3); @@ -722,7 +722,7 @@ static size_t HUF_decompress4X4_usingDTable_internal( if (op1 > opStart2) return ERROR(corruption_detected); if (op2 > opStart3) return ERROR(corruption_detected); if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ + /* note : op4 already verified within main loop */ /* finish bitStreams one by one */ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); @@ -848,9 +848,6 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); } - - //return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ - //return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ } size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) From 0cfe2ec2fdab88987e164e5e70d91e249fcd6270 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 20 Aug 2016 00:26:26 +0200 Subject: [PATCH 16/74] sync fse version --- lib/common/fse_decompress.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 918de64c..032e6577 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -68,6 +68,9 @@ #define FSE_isError ERR_isError #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + /* ************************************************************** * Complex types @@ -152,7 +155,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned position = (position + step) & tableMask; while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ } } - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -169,7 +171,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned } - #ifndef FSE_COMMONDEFS_ONLY /*-******************************************************* @@ -234,8 +235,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( FSE_DState_t state2; /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSE_isError(errorCode)) return errorCode; } + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); FSE_initDState(&state1, &bitD, dt); FSE_initDState(&state2, &bitD, dt); @@ -243,7 +243,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) /* 4 symbols per loop */ - for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ @@ -266,18 +266,14 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ while (1) { if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { *op++ = FSE_GETSYMBOL(&state2); break; } if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { *op++ = FSE_GETSYMBOL(&state1); break; @@ -320,8 +316,7 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr cSrcSize -= NCountLength; } - { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; } + CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ } From 1bee2d5e08208b01e496efa55c36c28422eaf35f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 20 Aug 2016 02:59:04 +0200 Subject: [PATCH 17/74] slight decompression speed improvement --- lib/decompress/zstd_decompress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6c449fa5..1a1d7ce3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -678,7 +678,9 @@ size_t ZSTD_execSequence(BYTE* op, if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ /* copy Literals */ - ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ From 8baf78a291532963a6bb47163ff7c064804ea341 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 20 Aug 2016 03:47:49 +0200 Subject: [PATCH 18/74] minor coding style --- NEWS | 4 ++++ lib/decompress/zstd_decompress.c | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index 015fd5e1..70429644 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v0.8.2 +API : ZDICT_getDictID() +Small decompression speed improvement + v0.8.1 New streaming API Changed : --ultra now enables levels beyond 19 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1a1d7ce3..2311719f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -590,9 +590,9 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) { seq_t seq; - U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); - U32 const mlCode = FSE_peekSymbol(&(seqState->stateML)); - U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ U32 const llBits = LL_bits[llCode]; U32 const mlBits = ML_bits[mlCode]; @@ -621,8 +621,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (!ofCode) offset = 0; else { - offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); + offset = OF_base[ofCode] + BIT_readBits(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } if (ofCode <= 1) { @@ -643,18 +643,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) seq.offset = offset; } - seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); - seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&seqState->DStream, llBits) : 0); /* <= 16 bits */ if (MEM_32bits() || - (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); /* ANS state update */ - FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ - FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ - FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ return seq; } From 70c2326bf194121a1213476a3d6b90ecb3ffa802 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 21 Aug 2016 00:24:18 +0200 Subject: [PATCH 19/74] spec : clarify last literal copy command --- zstd_compression_format.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 0989912d..61f6a57b 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -443,7 +443,7 @@ using little-endian convention. In this representation, bits on the left are smallest bits. -__`Literals_Block_Type`__ +__`Literals_Block_Type`__ This field uses 2 lowest bits of first byte, describing 4 different block types : @@ -460,7 +460,7 @@ This field uses 2 lowest bits of first byte, describing 4 different block types using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. -__`Size_Format`__ +__`Size_Format`__ `Size_Format` is divided into 2 families : @@ -697,8 +697,13 @@ A match copy command specifies an offset and a length. The offset gives the position to copy from, which can be within a previous block. -There are 3 symbol types, literals lengths, offsets and match lengths, -which are encoded together, interleaved in a single _bitstream_. +When all _sequences_ are decoded, +if there is any literal left in the _literal section_, +these bytes are added at the end of the block. + +The _Sequences_Section_ regroup all symbols required to decode commands. +There are 3 symbol types : literals lengths, offsets and match lengths. +They are encoded together, interleaved, in a single _bitstream_. Each symbol is a _code_ in its own context, which specifies a baseline and a number of bits to add. @@ -905,8 +910,8 @@ since it will be discovered and reported by the decoding process. The bitstream starts by reporting on which scale it operates. `Accuracy_Log = low4bits + 5`. -Note that maximum `Accuracy_Log` for literal and match length is `9`, -and for offsets it is `8`. Higher values are considered errors. +Note that maximum `Accuracy_Log` for literal and match lengths is `9`, +and for offsets is `8`. Higher values are considered errors. Then follow each symbol value, from `0` to last present one. The number of bits used by each field is variable. @@ -1128,8 +1133,8 @@ _Reserved ranges :_ However, for public distribution of compressed frames, the following ranges are reserved for future use and should not be used : - - low range : 1 - 32767 - - high range : >= (2^31) + - low range : 1 - 32767 + - high range : >= (2^31) __`Entropy_Tables`__ : following the same format as a [compressed blocks]. They are stored in following order : From d1733f741748e23b450af4e4946388e8d7a97954 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 21 Aug 2016 01:04:46 +0200 Subject: [PATCH 20/74] fixed crc bug in rare timing conditions within bench.c --- lib/compress/zstd_opt.h | 6 +++--- programs/bench.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index f30cdbfd..612ac3b4 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -529,7 +529,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; @@ -610,12 +610,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); mlen++; - } } } // for (cur = 1; cur <= last_pos; cur++) + } } } // for (cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; diff --git a/programs/bench.c b/programs/bench.c index 12c59bff..dc36d170 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -257,7 +257,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #if 1 /* Decompression */ - memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ UTIL_waitForNextTick(ticksPerSecond); From cb3276329aae840245fbef0b8f6dce141ea37da4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Aug 2016 00:30:31 +0200 Subject: [PATCH 21/74] added sizeof CStream and DStream --- lib/compress/zstd_compress.c | 4 ++++ lib/decompress/zstd_decompress.c | 18 +++++++++--------- lib/zstd.h | 4 +++- tests/fuzzer.c | 5 ++++- tests/zstreamtest.c | 20 +++++++++++++++++--- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 00b1408c..bc74da09 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2867,6 +2867,10 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); } +size_t ZSTD_sizeofCStream(const ZSTD_CStream* zcs) +{ + return sizeof(zcs) + ZSTD_sizeofCCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; +} /*====== Compression ======*/ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 2311719f..c62ef199 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -980,9 +980,10 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr } -/*-********************************** -* Streaming Decompression API -************************************/ +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -1382,6 +1383,11 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_sizeofDStream(const ZSTD_DStream* zds) +{ + return sizeof(*zds) + ZSTD_sizeofDCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; +} + /* *** Decompression *** */ @@ -1531,9 +1537,3 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return nextSrcSizeHint; } } - - - - - - diff --git a/lib/zstd.h b/lib/zstd.h index d07cc3c7..ab5b4f58 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -431,13 +431,15 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +size_t ZSTD_sizeofCStream(const ZSTD_CStream* zcs); + /*====== decompression ======*/ /* advanced */ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); - +size_t ZSTD_sizeofDStream(const ZSTD_DStream* zds); /* ****************************************************************** diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 87647b9d..bed86f59 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -100,7 +100,6 @@ static unsigned FUZ_rand(unsigned* src) return rand32 >> 5; } - static unsigned FUZ_highbit32(U32 v32) { unsigned nbBits = 0; @@ -110,6 +109,10 @@ static unsigned FUZ_highbit32(U32 v32) } +/*============================================= +* Basic Unit tests +=============================================*/ + #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error #define CHECK(fn) { CHECK_V(err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 7f136b1b..4448f23c 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -61,7 +61,6 @@ static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; - /*-************************************ * Display Macros **************************************/ @@ -93,7 +92,6 @@ static U32 FUZ_GetMilliStart(void) return nCount; } - static U32 FUZ_GetMilliSpan(U32 nTimeStart) { U32 const nCurrent = FUZ_GetMilliStart(); @@ -117,7 +115,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr) return rand32 >> 5; } - /* static unsigned FUZ_highbit32(U32 v32) { @@ -141,6 +138,11 @@ static void freeFunction(void* opaque, void* address) free(address); } + +/*====================================================== +* Basic Unit tests +======================================================*/ + static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { int testResult = 0; @@ -187,6 +189,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo cSize += outBuff.pos; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++); + { size_t const s = ZSTD_sizeofCStream(zc); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); + } + /* skippable frame test */ DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); @@ -218,6 +226,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : check DStream size : ", testNb++); + { size_t const s = ZSTD_sizeofDStream(zd); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); + } + /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); { size_t r = 1; From 70e3b3130687839a36383e788d39ce1733426e20 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Aug 2016 01:18:06 +0200 Subject: [PATCH 22/74] fixed playtests on os-x --- lib/compress/zstd_compress.c | 6 ++-- lib/decompress/zstd_decompress.c | 6 ++-- lib/zstd.h | 48 ++++++++++++++++---------------- tests/playTests.sh | 2 +- tests/zstreamtest.c | 4 +-- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bc74da09..a696b2bc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -154,7 +154,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) return 0; /* reserved as a potential error code in the future */ } -size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx) +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { return sizeof(*cctx) + cctx->workSpaceSize; } @@ -2867,9 +2867,9 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); } -size_t ZSTD_sizeofCStream(const ZSTD_CStream* zcs) +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { - return sizeof(zcs) + ZSTD_sizeofCCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c62ef199..5cd9deb5 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -138,7 +138,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ -size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } @@ -1383,9 +1383,9 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } -size_t ZSTD_sizeofDStream(const ZSTD_DStream* zds) +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeofDCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; } diff --git a/lib/zstd.h b/lib/zstd.h index ab5b4f58..cb667484 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -245,16 +245,16 @@ typedef struct ZSTD_outBuffer_s { * *******************************************************************/ typedef struct ZSTD_CStream_s ZSTD_CStream; -ZSTD_CStream* ZSTD_createCStream(void); -size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); -size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ -size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); /*====== decompression ======*/ @@ -281,14 +281,14 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * *******************************************************************************/ typedef struct ZSTD_DStream_s ZSTD_DStream; -ZSTD_DStream* ZSTD_createDStream(void); -size_t ZSTD_freeDStream(ZSTD_DStream* zds); +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); -size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ -size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); @@ -376,12 +376,12 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS /*! ZSTD_sizeofCCtx() : * Gives the amount of memory used by a given ZSTD_CCtx */ -ZSTDLIB_API size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. @@ -418,7 +418,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); /*! ZSTD_sizeofDCtx() : * Gives the amount of memory used by a given ZSTD_DCtx */ -ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** @@ -427,19 +427,19 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /*====== compression ======*/ -ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); -size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); -size_t ZSTD_sizeofCStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /*====== decompression ======*/ /* advanced */ -ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); -size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); -size_t ZSTD_sizeofDStream(const ZSTD_DStream* zds); +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); /* ****************************************************************** diff --git a/tests/playTests.sh b/tests/playTests.sh index 013ba6f6..75b334ff 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -32,7 +32,7 @@ case "$OS" in esac MD5SUM="md5sum" -if [ "$TRAVIS_OS_NAME" = "osx" ]; then +if [[ "$OSTYPE" == "darwin"* ]]; then MD5SUM="md5 -r" fi diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 4448f23c..4c81e8df 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -190,7 +190,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++); - { size_t const s = ZSTD_sizeofCStream(zc); + { size_t const s = ZSTD_sizeof_CStream(zc); if (ZSTD_isError(s)) goto _output_error; DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); } @@ -227,7 +227,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : check DStream size : ", testNb++); - { size_t const s = ZSTD_sizeofDStream(zd); + { size_t const s = ZSTD_sizeof_DStream(zd); if (ZSTD_isError(s)) goto _output_error; DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); } From 3071c3e3034e7b506a0bb311b907b9eb02d56910 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Aug 2016 01:34:34 +0200 Subject: [PATCH 23/74] STREAM_WINDOW_MAX : protect streaming from unreasonable memory requirements --- lib/decompress/zstd_decompress.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5cd9deb5..524c2f56 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -50,6 +50,16 @@ #endif +/*! +* STREAM_WINDOW_MAX : +* maximum window size accepted by DStream. +* frames requiring more memory will be rejected. +*/ +#ifndef ZSTD_STREAM_WINDOW_MAX +# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ +#endif + + /*-******************************************************* * Dependencies *********************************************************/ @@ -1445,6 +1455,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; + if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported); zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); From b62e696992bdaaed06d04368022f74e17b23e02d Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 23 Aug 2016 13:54:37 +0200 Subject: [PATCH 24/74] fixed typo in test-zstd-speed.py --- tests/test-zstd-speed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index a0e1a281..dba4f855 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -192,7 +192,7 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd') md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32') md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang') - print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd32_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang)) + print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang)) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" From 17e482efdddf90c21b51fe4df545eed3dfe6b9ab Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 23 Aug 2016 16:58:10 +0200 Subject: [PATCH 25/74] added ZSTD_setDStreamParameter() --- lib/common/error_private.h | 1 + lib/common/error_public.h | 4 ++++ lib/decompress/zstd_decompress.c | 26 ++++++++++++++++++++------ lib/zstd.h | 10 ++++++---- tests/zstreamtest.c | 23 +++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 88906149..34c0c4c0 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 29050b3b..10f020cf 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -37,6 +37,9 @@ extern "C" { #endif +/*===== dependency =====*/ +#include /* size_t */ + /* **************************************** * error codes list @@ -45,6 +48,7 @@ typedef enum { ZSTD_error_no_error, ZSTD_error_GENERIC, ZSTD_error_prefix_unknown, + ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, ZSTD_error_compressionParameter_unsupported, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 524c2f56..bbeca0d8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -51,12 +51,12 @@ /*! -* STREAM_WINDOW_MAX : -* maximum window size accepted by DStream. -* frames requiring more memory will be rejected. +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. */ -#ifndef ZSTD_STREAM_WINDOW_MAX -# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */ #endif @@ -1329,6 +1329,7 @@ struct ZSTD_DStream_s { char* inBuff; size_t inBuffSize; size_t inPos; + size_t maxWindowSize; char* outBuff; size_t outBuffSize; size_t outStart; @@ -1362,6 +1363,7 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) zds->zd = ZSTD_createDCtx_advanced(customMem); if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; + zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; } @@ -1393,6 +1395,18 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + switch(paramType) + { + default : return ERROR(parameter_unknown); + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + } + return 0; +} + + size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; @@ -1451,11 +1465,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; - if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported); zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); diff --git a/lib/zstd.h b/lib/zstd.h index cb667484..fd6170c8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -87,7 +87,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * When `return==0`, data to decompress can have any size. * In which case, it's necessary to use streaming mode to decompress data. * Optionally, application may rely on its own implied limits. -* (For example, application own data could be necessarily cut into blocks <= 16 KB). +* (For example, application data could be necessarily cut into blocks <= 16 KB). * note 3 : decompressed size could be wrong or intentionally modified ! * Always ensure result fits within application's authorized limits ! * Each application can set its own limits. @@ -264,7 +264,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-init multiple times. +* ZSTD_DStream objects can be re-used multiple times. * * Use ZSTD_initDStream() to start a new decompression operation, * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. @@ -422,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Advanced Streaming +* Advanced Streaming functions ********************************************************************/ /*====== compression ======*/ @@ -436,9 +436,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /*====== decompression ======*/ -/* advanced */ +typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; + ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 4c81e8df..edfb07eb 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -212,6 +212,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */ + if (ZSTD_isError(r)) goto _output_error; } { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ @@ -267,6 +269,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++); + { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */ + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + /* Memory restriction */ + DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000); /* too small limit */ + if (ZSTD_isError(r)) goto _output_error; } + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ + DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); } + + _end: ZSTD_freeCStream(zc); ZSTD_freeDStream(zd); From c54692faeb77ec2e67a6c627ce5093d98f501a08 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Aug 2016 01:10:42 +0200 Subject: [PATCH 26/74] improved level 3 --- lib/compress/zstd_compress.c | 45 ++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a696b2bc..e9acead8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1232,9 +1232,20 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - offset = (U32)(ip-match); - while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* match3 = base + matchIndex3; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, match3+8, iend) + 8; + ip++; + offset = (U32)(ip-match3); + while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } } else { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; @@ -1361,16 +1372,32 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, offset_2 = offset_1; offset_1 = offset; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { - const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; U32 offset; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset = current - matchIndex; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; + ip++; + offset = current+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + offset = current - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } offset_2 = offset_1; offset_1 = offset; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; @@ -3043,7 +3070,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ - { 20, 16, 18, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ { 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/ { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ From 24b68a5b23225a368454876d21896d042af3acdd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Aug 2016 14:22:26 +0200 Subject: [PATCH 27/74] update cLevel table for 256KB --- lib/compress/zstd_compress.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e9acead8..7e6dd7ac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3093,24 +3093,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ - { 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ - { 18, 13, 14, 1, 6, 4, ZSTD_fast }, /* level 1 */ - { 18, 15, 17, 1, 5, 4, ZSTD_fast }, /* level 2 */ - { 18, 13, 15, 1, 5, 4, ZSTD_greedy }, /* level 3.*/ - { 18, 15, 17, 1, 5, 4, ZSTD_greedy }, /* level 4.*/ - { 18, 16, 17, 4, 5, 4, ZSTD_greedy }, /* level 5 */ - { 18, 17, 17, 5, 5, 4, ZSTD_greedy }, /* level 6 */ - { 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 18, 17, 17, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */ - { 18, 17, 17, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */ - { 18, 17, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 11.*/ - { 18, 18, 17, 7, 4, 4, ZSTD_lazy2 }, /* level 12.*/ - { 18, 19, 17, 7, 4, 4, ZSTD_btlazy2 }, /* level 13 */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 18, 13, 14, 0, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ + { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ + { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 18, 8, 4, 24, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 18, 8, 3, 48, ZSTD_btopt }, /* level 16.*/ - { 18, 19, 18, 8, 3, 96, ZSTD_btopt }, /* level 17.*/ + { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ { 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/ From 57ef4b1a0d82335e1a599a8ec9ea9f07703406ab Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 17:16:56 +0200 Subject: [PATCH 28/74] zstd_v07.c: removed unused macros --- lib/legacy/zstd_v06.c | 13 ------------- lib/legacy/zstd_v07.c | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 3dcb0d97..ae10d153 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -669,19 +669,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code) /*-************************************* * Common constants ***************************************/ -#define ZSTDv06_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs -#include -#if defined(ZSTDv06_OPT_DEBUG) && ZSTDv06_OPT_DEBUG>=9 - #define ZSTDv06_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTDv06_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTDv06_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTDv06_LOG_PARSER(...) - #define ZSTDv06_LOG_ENCODE(...) - #define ZSTDv06_LOG_BLOCK(...) -#endif - -#define ZSTDv06_OPT_NUM (1<<12) #define ZSTDv06_DICT_MAGIC 0xEC30A436 #define ZSTDv06_REP_NUM 3 diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 2e8ffa45..591a3c15 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -2989,18 +2989,6 @@ void ZSTDv07_defaultFreeFunction(void* opaque, void* address) /*-************************************* * Common constants ***************************************/ -#define ZSTDv07_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#include -#if defined(ZSTDv07_OPT_DEBUG) && ZSTDv07_OPT_DEBUG>=9 - #define ZSTDv07_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTDv07_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTDv07_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTDv07_LOG_PARSER(...) - #define ZSTDv07_LOG_ENCODE(...) - #define ZSTDv07_LOG_BLOCK(...) -#endif - #define ZSTDv07_OPT_NUM (1<<12) #define ZSTDv07_DICT_MAGIC 0xEC30A437 /* v0.7 */ From 8a36f8527cec8c9470640afef07e4c3222c63c85 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 17:19:12 +0200 Subject: [PATCH 29/74] removed stats in debug mode --- lib/common/zstd_internal.h | 25 +---- lib/compress/.debug/zstd_stats.h | 162 ------------------------------- lib/compress/zstd_compress.c | 6 -- lib/compress/zstd_opt.h | 47 --------- 4 files changed, 3 insertions(+), 237 deletions(-) delete mode 100644 lib/compress/.debug/zstd_stats.h diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 7dc843e3..fc07a79d 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -51,18 +51,9 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 - #include - #include - #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTD_LOG_PARSER(...) - #define ZSTD_LOG_ENCODE(...) - #define ZSTD_LOG_BLOCK(...) -#endif +#define ZSTD_LOG_PARSER(...) +#define ZSTD_LOG_ENCODE(...) +#define ZSTD_LOG_BLOCK(...) #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ @@ -184,16 +175,6 @@ typedef struct { U32 rep[ZSTD_REP_NUM]; } ZSTD_optimal_t; -#if ZSTD_OPT_DEBUG == 3 - #include ".debug/zstd_stats.h" -#else - struct ZSTD_stats_s { U32 unused; }; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -#endif /* #if ZSTD_OPT_DEBUG == 3 */ - typedef struct seqDef_s { U32 offset; diff --git a/lib/compress/.debug/zstd_stats.h b/lib/compress/.debug/zstd_stats.h deleted file mode 100644 index 5cefa311..00000000 --- a/lib/compress/.debug/zstd_stats.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - zstd - standard compression library - Header File for static linking only - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ -#ifndef ZSTD_STATS_H -#define ZSTD_STATS_H - - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-************************************* -* Types -***************************************/ -struct ZSTD_stats_s { - U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength; - U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; - U32 litSum, matchLengthSum, litLengthSum, offCodeSum; - U32 matchLengthFreq[MaxML+1]; - U32 litLengthFreq[MaxLL+1]; - U32 litFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); - printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); - printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8); -} - - -MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) -{ - stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; - stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0; -} - - -MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) -{ - unsigned u; - - stats->litSum = (2<litLengthSum = MaxLL+1; - stats->matchLengthSum = MaxML+1; - stats->offCodeSum = (MaxOff+1); - - for (u=0; u<=MaxLit; u++) - stats->litFreq[u] = 1; - for (u=0; u<=MaxLL; u++) - stats->litLengthFreq[u] = 1; - for (u=0; u<=MaxML; u++) - stats->matchLengthFreq[u] = 1; - for (u=0; u<=MaxOff; u++) - stats->offCodeFreq[u] = 1; -} - - -MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) -{ - U32 u; - /* literals */ - stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); - for (u=0; u < litLength; u++) - stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); - stats->litSum += litLength; - for (u=0; u < litLength; u++) - stats->litFreq[literals[u]]++; - - /* literal Length */ - { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - if (litLength) { - stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1); - } else { - stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); - } - stats->litLengthFreq[llCode]++; - stats->litLengthSum++; - } - - /* match offset */ - { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); - stats->priceOffset += offCode; - stats->offCodeSum++; - stats->offCodeFreq[offCode]++; - } - - /* match Length */ - { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1); - stats->matchLengthFreq[mlCode]++; - stats->matchLengthSum++; - } - - if (offset == 0) stats->totalRepSum++; - stats->totalSeqSum++; - stats->totalMatchSum += matchLength; - stats->totalLitSum += litLength; -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTD_STATIC_H */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a696b2bc..826c5f97 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -765,8 +765,6 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif - ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode); /* debug only */ - /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); seqStorePtr->lit += litLength; @@ -1945,7 +1943,6 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; - ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -2236,7 +2233,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; ZSTD_stats_t* stats = &cctx->seqStore.stats; - ZSTD_statsInit(stats); /* debug only */ if (cctx->params.fParams.checksumFlag) XXH64_update(&cctx->xxhState, src, srcSize); @@ -2244,7 +2240,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, while (remaining) { U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); size_t cSize; - ZSTD_statsResetFreqs(stats); /* debug only */ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; @@ -2293,7 +2288,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; - ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */ return op-ostart; } diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 612ac3b4..1a37101a 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -295,23 +295,10 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { match = base + matchIndex; if (match[matchLength] == ip[matchLength]) { -#if ZSTD_OPT_DEBUG >= 5 - size_t ml; - if (matchIndex < dictLimit) - ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart); - else - ml = ZSTD_count(ip, match, ip+matchLength); - if (ml < matchLength) - printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0); -#endif matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; } } else { match = dictBase + matchIndex; -#if ZSTD_OPT_DEBUG >= 5 - if (memcmp(match, ip, matchLength) != 0) - printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0); -#endif matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match); if (matchIndex+matchLength >= dictLimit) @@ -672,20 +659,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset+1 > ZSTD_REP_MOVE_OPT) - ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend); - else - ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; @@ -990,26 +963,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset+1 > ZSTD_REP_MOVE_OPT) { - best_off = offset+1 - ZSTD_REP_MOVE_OPT; - if (best_off > (size_t)(ip - prefixStart)) { - const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); - ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); - } - else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend); - } - else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; From 83388e109f54ecdc23efaf7256c93ad733a85c0b Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 17:22:20 +0200 Subject: [PATCH 30/74] removed ZSTD_LOG_PARSER --- lib/common/zstd_internal.h | 1 - lib/compress/zstd_opt.h | 37 ------------------------------------- 2 files changed, 38 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index fc07a79d..3518ade6 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -51,7 +51,6 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_LOG_PARSER(...) #define ZSTD_LOG_ENCODE(...) #define ZSTD_LOG_BLOCK(...) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 1a37101a..beef30d2 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -199,7 +199,6 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B opt[pos].off = offset_; \ opt[pos].litlen = litlen_; \ opt[pos].price = price_; \ - ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -300,7 +299,6 @@ static U32 ZSTD_insertBtAndGetAllMatches ( } else { match = dictBase + matchIndex; matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match); if (matchIndex+matchLength >= dictLimit) match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } @@ -445,7 +443,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart)) && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; - ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; @@ -461,7 +458,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); - ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { @@ -477,7 +473,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) @@ -528,8 +523,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } - ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - best_mlen = minMatch; { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; last_pos = cur + 1; - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -560,7 +551,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } if (mlen > best_mlen) best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) @@ -570,7 +560,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } } } match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); - ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { best_mlen = matches[match_num-1].len; @@ -584,7 +573,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; @@ -597,7 +585,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); @@ -610,10 +597,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* store sequence */ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); - opt[0].mlen = 1; while (1) { @@ -628,12 +611,10 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } for (u = 0; u <= last_pos;) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); u += opt[u].mlen; } for (cur=0; cur < last_pos; ) { - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -737,7 +718,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; goto _storeSequence; @@ -755,7 +735,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ - ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } { U32 i; for (i=0; i0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); @@ -817,15 +795,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } - ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); best_mlen = 0; { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); @@ -840,11 +815,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; last_pos = cur + 1; - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -861,7 +834,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) @@ -871,7 +843,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } } } match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); if (match_num > 0 && matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -887,7 +858,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); while (mlen <= best_mlen) { if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; @@ -900,7 +870,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); @@ -913,10 +882,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* store sequence */ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); - opt[0].mlen = 1; while (1) { @@ -931,12 +896,10 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } for (u = 0; u <= last_pos; ) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); u += opt[u].mlen; } for (cur=0; cur < last_pos; ) { - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; From 4e90f6c1e084d72012d4354a6f0b2feada3c9c25 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 17:24:11 +0200 Subject: [PATCH 31/74] removed ZSTD_LOG_ENCODE and ZSTD_LOG_BLOCK --- lib/common/zstd_internal.h | 3 --- lib/compress/zstd_compress.c | 5 ----- lib/compress/zstd_opt.h | 16 ---------------- 3 files changed, 24 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 3518ade6..47d7d8ec 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -51,9 +51,6 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_LOG_ENCODE(...) -#define ZSTD_LOG_BLOCK(...) - #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 826c5f97..0a8dde39 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2396,7 +2396,6 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const { size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); - ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", cctx->base, cctx->params.cParams.searchLength); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); } @@ -2554,14 +2553,12 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); } size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) { - ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel); } @@ -2647,13 +2644,11 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co { ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize); params.fParams.contentSizeFlag = 1; - ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { - ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); } diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index beef30d2..e3c5cf2e 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -426,8 +426,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, { U32 i; for (i=0; irep[i]; } inr = ip; - ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); - /* Match Loop */ while (ip < ilimit) { U32 cur, match_num, last_pos, litlen, price; @@ -515,12 +513,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } best_mlen = minMatch; @@ -620,7 +616,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ offset = opt[cur].off; cur += mlen; litLength = (U32)(ip - anchor); - // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; @@ -634,12 +629,9 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ rep[1] = rep[0]; rep[0] = best_off; } - if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; if (litLength==0) offset--; } - ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; @@ -650,7 +642,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ /* Last Literals */ { size_t const lastLLSize = iend - anchor; - ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)lastLLSize); memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } @@ -692,8 +683,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ip += (ip==prefixStart); inr = ip; - ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); - /* Match Loop */ while (ip < ilimit) { U32 cur, match_num, last_pos, litlen, price; @@ -905,7 +894,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ offset = opt[cur].off; cur += mlen; litLength = (U32)(ip - anchor); - // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); if (offset > ZSTD_REP_MOVE_OPT) { rep[2] = rep[1]; @@ -920,12 +908,9 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ rep[0] = best_off; } - if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); }; if (litLength==0) offset--; } - ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); anchor = ip = ip + mlen; @@ -936,7 +921,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ /* Last Literals */ { size_t lastLLSize = iend - anchor; - ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)(lastLLSize)); memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } From e416e30019ce4c25d99fe043c194e38c2456abda Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 17:32:09 +0200 Subject: [PATCH 32/74] remove unnecessary comments --- lib/common/zstd_internal.h | 1 - lib/compress/zstd_compress.c | 1 - lib/compress/zstd_opt.h | 10 ++++------ lib/decompress/zstd_decompress.c | 1 - programs/util.h | 9 +++------ zlibWrapper/zstd_zlibwrapper.c | 6 +----- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 47d7d8ec..8b9b6290 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -210,7 +210,6 @@ typedef struct { U32 cachedPrice; U32 cachedLitLength; const BYTE* cachedLiterals; - ZSTD_stats_t stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0a8dde39..a32cb5ad 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2232,7 +2232,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; - ZSTD_stats_t* stats = &cctx->seqStore.stats; if (cctx->params.fParams.checksumFlag) XXH64_update(&cctx->xxhState, src, srcSize); diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index e3c5cf2e..cb332d94 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -500,7 +500,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); } - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price <= opt[cur].price) SET_PRICE(cur, 1, 0, litlen, price); if (cur == last_pos) break; @@ -532,7 +532,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, goto _storeSequence; } - //best_off = ((i<=1) & (opt[cur].mlen != 1)) ? 1-i : i; best_off = i - (opt[cur].mlen != 1); if (opt[cur].mlen == 1) { @@ -585,7 +584,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); mlen++; - } } } // for (cur = 1; cur <= last_pos; cur++) + } } } best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; @@ -739,7 +738,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, best_mlen = (last_pos) ? last_pos : minMatch; - // set prices using matches at position = 0 + /* set prices using matches at position = 0 */ for (u = 0; u < match_num; u++) { mlen = (u>0) ? matches[u-1].len+1 : best_mlen; best_mlen = matches[u].len; @@ -752,7 +751,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } } if (last_pos < minMatch) { - // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ ip++; continue; } @@ -771,7 +769,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); } - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price <= opt[cur].price) SET_PRICE(cur, 1, 0, litlen, price); if (cur == last_pos) break; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 524c2f56..c3d9d608 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -64,7 +64,6 @@ * Dependencies *********************************************************/ #include /* memcpy, memmove, memset */ -#include /* debug only : printf */ #include "mem.h" /* low level memory routines */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64_* */ diff --git a/programs/util.h b/programs/util.h index 1e4c7977..37ae4cb2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -245,7 +245,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { if (strcmp (cFile.cFileName, "..") == 0 || strcmp (cFile.cFileName, ".") == 0) continue; - // printf ("[%s]\n", path); + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { FindClose(hFile); return 0; } } @@ -261,7 +261,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ *pos += pathLength + 1; nbFiles++; } - // printf ("%s\\%s nbFiles=%d left=%d\n", dirName, cFile.cFileName, nbFiles, (int)(bufEnd - *bufStart)); } } while (FindNextFile(hFile, &cFile)); @@ -297,7 +296,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ continue; } if (UTIL_isDirectory(path)) { - // printf ("[%s]\n", path); nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { closedir(dir); return 0; } } else { @@ -312,9 +310,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ *pos += pathLength + 1; nbFiles++; } - // printf ("%s/%s nbFiles=%d left=%d\n", dirName, entry->d_name, nbFiles, (int)(bufEnd - *bufStart)); } - errno = 0; // clear errno after UTIL_isDirectory, UTIL_prepareFileList + errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ } if (errno != 0) { @@ -335,7 +332,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return 0; } -#endif // #ifdef _WIN32 +#endif /* #ifdef _WIN32 */ /* * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index e23a235c..0218c6a2 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -44,7 +44,7 @@ #define ZWRAP_HEADERSIZE 4 #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ -#define LOG_WRAPPER(...) // printf(__VA_ARGS__) +#define LOG_WRAPPER(...) /* printf(__VA_ARGS__) */ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -236,8 +236,6 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } else { bytesLeft = ZBUFF_compressEnd(zwc->zbc, strm->next_out, &dstCapacity); LOG_WRAPPER("ZBUFF_compressEnd dstCapacity=%d bytesLeft=%d\n", (int)dstCapacity, (int)bytesLeft); - // { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, 1); - // if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } } if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; strm->next_out += dstCapacity; @@ -752,7 +750,6 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)) { if (sourceLen < 4 || MEM_readLE32(source) != ZSTD_MAGICNUMBER) -// if (!g_useZSTD) return uncompress(dest, destLen, source, sourceLen); { size_t dstCapacity = *destLen; @@ -856,7 +853,6 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) va_end (args); ret = gzprintf(file, buf); - // printf("gzprintf ret=%d\n", ret); return ret; } FINISH_WITH_GZ_ERR("gzprintf is not supported!"); From a2cdffe556850a1e0d56f547d6c855e3427df84a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 24 Aug 2016 19:42:15 +0200 Subject: [PATCH 33/74] fixed wrong parameter issue --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7e6dd7ac..0c13bc21 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3094,7 +3094,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ - { 18, 13, 14, 0, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ From c23a07830e8b63d610a6018a4a1e1bf8cd7975e7 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 24 Aug 2016 21:42:43 +0200 Subject: [PATCH 34/74] updated projects\README.md --- projects/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/projects/README.md b/projects/README.md index c2fa7478..96202c1c 100644 --- a/projects/README.md +++ b/projects/README.md @@ -5,6 +5,19 @@ projects for various integrated development environments (IDE) The following projects are included with the zstd distribution: - cmake - CMake project contributed by Artyom Dymchenko +- VS2005 - Visual Studio 2005 project - VS2008 - Visual Studio 2008 project - VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) - build - command line scripts prepared for Visual Studio compilation without IDE + + +#### How to compile zstd with Visual Studio + +1. Install Visual Studio e.g. VS 2015 Community Edition (it's free). +2. Download the latest version of zstd from https://github.com/Cyan4973/zstd/releases +3. Decompress ZIP archive. +4. Go to decompressed directory then to "projects" then "VS2010" and open zstd.sln +5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree. +6. Change "Debug" to "Release" and if you have 64-bit Windows change also "Win32" to "x64". +7. Press F7 on keyboard or select "BUILD" from the menu bar and choose "Build Solution". +8. If compilation will be fine a compiled executable will be in projects\VS2010\bin\x64\Release\zstd.exe From 7f18c6e5ef36e0c0e4921384c45502d726ac13fb Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 08:47:18 +0200 Subject: [PATCH 35/74] improved tests/README.md --- projects/README.md | 8 ++++---- tests/Makefile | 10 +++++++--- tests/README.md | 13 ++++++++++++- tests/zbufftest.c | 2 +- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/projects/README.md b/projects/README.md index 96202c1c..d724cdcb 100644 --- a/projects/README.md +++ b/projects/README.md @@ -16,8 +16,8 @@ The following projects are included with the zstd distribution: 1. Install Visual Studio e.g. VS 2015 Community Edition (it's free). 2. Download the latest version of zstd from https://github.com/Cyan4973/zstd/releases 3. Decompress ZIP archive. -4. Go to decompressed directory then to "projects" then "VS2010" and open zstd.sln +4. Go to decompressed directory then to `projects` then `VS2010` and open `zstd.sln` 5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree. -6. Change "Debug" to "Release" and if you have 64-bit Windows change also "Win32" to "x64". -7. Press F7 on keyboard or select "BUILD" from the menu bar and choose "Build Solution". -8. If compilation will be fine a compiled executable will be in projects\VS2010\bin\x64\Release\zstd.exe +6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`. +7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`. +8. If compilation will be fine a compiled executable will be in `projects\VS2010\bin\x64\Release\zstd.exe` diff --git a/tests/Makefile b/tests/Makefile index a8fb97c2..c054bbbb 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -22,13 +22,17 @@ # - zstd homepage : http://www.zstd.net/ # ########################################################################## # datagen : Synthetic and parametrable data generator, for tests +# fullbench : Precisely measure speed for each zstd inner functions +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode # fuzzer : Test tool, to check zstd integrity on target platform # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# paramgrill : parameter tester for zstd +# test-zstd-speed.py : script for testing zstd speed difference between commits +# versionsTest : compatibility test between zstd versions stored on Github (v0.1+) # zbufftest : Test tool, to check ZBUFF integrity on target platform # zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode -# fullbench : Precisely measure speed for each zstd inner function -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode -# versionstest : Compatibility test between zstd versions stored on Github (v0.1+) +# zstreamtest : Fuzzer test tool for zstd streaming API +# zbufftest32: Same as zstreamtest, but forced to compile in 32-bits mode # ########################################################################## DESTDIR?= diff --git a/tests/README.md b/tests/README.md index 7b379541..309638f9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,17 @@ -scripts for automated testing of zstd +programs and scripts for automated testing of zstd ================================ +This directory contains the following programs and scripts: +- datagen : Synthetic and parametrable data generator, for tests +- fullbench : Precisely measure speed for each zstd inner functions +- fuzzer : Test tool, to check zstd integrity on target platform +- paramgrill : parameter tester for zstd +- test-zstd-speed.py : script for testing zstd speed difference between commits +- test-zstd-versions.py : compatibility test between zstd versions stored on Github (v0.1+) +- zbufftest : Test tool, to check ZBUFF integrity on target platform +- zstreamtest : Fuzzer test tool for zstd streaming API + + #### test-zstd-versions.py - script for testing zstd interoperability between versions This script creates `versionsTest` directory to which zstd repository is cloned. diff --git a/tests/zbufftest.c b/tests/zbufftest.c index ce6beb24..151785c6 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -1,5 +1,5 @@ /* - Fuzzer test tool for zstd_buffered + Fuzzer test tool for ZBUFF - a buffered streaming API for ZSTD Copyright (C) Yann Collet 2015-2016 GPL v2 License From ab2f7708c132aa85b3cca59698829a41baf7ac86 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 10:07:20 +0200 Subject: [PATCH 36/74] created programs\README.md --- programs/Makefile | 6 +-- programs/README.md | 93 ++++++++++++++++++++++++++++++++++++++++++++++ projects/README.md | 10 ++--- tests/README.md | 20 +++++----- 4 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 programs/README.md diff --git a/programs/Makefile b/programs/Makefile index d738f40c..dd6e7838 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -24,9 +24,9 @@ # zstd : Command Line Utility, supporting gzip-like arguments # zstd32 : Same as zstd, but forced to compile in 32-bits mode # zstd_nolegacy : zstd without support of decompression of legacy versions -# zstd-small: minimal zstd without dictBuilder and bench -# zstd-compress: compressor-only version of zstd -# zstd-decompress: decompressor-only version of zstd +# zstd-small : minimal zstd without dictionary builder and benchmark +# zstd-compress : compressor-only version of zstd +# zstd-decompress : decompressor-only version of zstd # ########################################################################## DESTDIR?= diff --git a/programs/README.md b/programs/README.md new file mode 100644 index 00000000..f38e6c49 --- /dev/null +++ b/programs/README.md @@ -0,0 +1,93 @@ +zstd - Command Line Interface +================================ + +Command Line Interface (CLI) can be created using the `make` command without any additional parameters. +There are however other Makefile targets that create different variations of CLI: +- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd versions +- `zstd32` : Same as `zstd`, but forced to compile in 32-bits mode +- `zstd_nolegacy` : Same as `zstd` except of support for decompression of legacy zstd versions +- `zstd-small` : CLI optimized for minimal size; without dictionary builder, benchmark, and support for decompression of legacy zstd versions +- `zstd-compress` : compressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions +- `zstd-decompress` : decompressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions + + +#### Aggregation of parameters +CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. + + +#### Dictionary builder in Command Line Interface +Zstd offers a training mode, which can be used to tune the algorithm for a selected +type of data, by providing it with a few samples. The result of the training is stored +in a file selected with the `-o` option (default name is `dictionary`), +which can be loaded before compression and decompression. + +Using a dictionary, the compression ratio achievable on small data improves dramatically. +These compression gains are achieved while simultaneously providing faster compression and decompression speeds. +Dictionary work if there is some correlation in a family of small data (there is no universal dictionary). +Hence, deploying one dictionary per type of data will provide the greater benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm +will rely more and more on previously decoded content to compress the rest of the file. + +Usage of the dictionary builder and created dictionaries with CLI: +1) Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName` +2) Compress with dictionary: `zstd FILE -D dictionaryName` +3) Decompress with dictionary: `zstd --decompress FILE.zst -D dictionaryName` + + + +#### Benchmark in Command Line Interface +CLI includes in-memory compression benchmark module for zstd. +The benchmark is conducted using given filenames which are read into memory and joined together. +It makes benchmark more precise as it eliminates I/O overhead. +Many filenames can be supplied as multiple parameters, parameters with wildcards or +names of directories can be used as parameters with the `-r` option. + +The benchmark measures ratio, compressed size, compression and decompression speed. +One can select compression levels starting from `-b` and ending with `-e`. +The `-i` parameter selects minimal time used for each of tested levels. + + + +#### Usage of Command Line Interface +The full list of options can be obtained with `-h` or `-H` parameter: +``` +Usage : + zstd [args] [FILE(s)] [-o file] + +FILE : a filename + with no FILE, or when FILE is - , read standard input +Arguments : + -# : # compression level (1-19, default:3) + -d : decompression + -D file: use `file` as Dictionary + -o file: result stored into `file` (only if 1 input file) + -f : overwrite output without prompting +--rm : remove source file(s) after successful de/compression + -k : preserve source file(s) (default) + -h/-H : display help/long help and exit + +Advanced arguments : + -V : display Version number and exit + -v : verbose mode; specify multiple times to increase log level (default:2) + -q : suppress warnings; specify twice to suppress errors too + -c : force write to standard output, even if it is the console + -r : operate recursively on directories +--ultra : enable levels beyond 19, up to 22 (requires more memory) +--no-dictID : don't write dictID into header (dictionary compression) +--[no-]check : integrity check (default:enabled) +--test : test compressed file integrity +--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) + +Dictionary builder : +--train ## : create a dictionary from a training set of files + -o file : `file` is dictionary name (default: dictionary) +--maxdict ## : limit dictionary to specified size (default : 112640) + -s# : dictionary selectivity level (default: 9) +--dictID ## : force dictionary ID to specified value (default: random) + +Benchmark arguments : + -b# : benchmark file(s), using # compression level (default : 1) + -e# : test all compression levels from -bX to # (default: 1) + -i# : minimum evaluation time in seconds (default : 3s) + -B# : cut file into independent blocks of size # (default: no block) + ``` \ No newline at end of file diff --git a/projects/README.md b/projects/README.md index d724cdcb..6353623b 100644 --- a/projects/README.md +++ b/projects/README.md @@ -4,11 +4,11 @@ projects for various integrated development environments (IDE) #### Included projects The following projects are included with the zstd distribution: -- cmake - CMake project contributed by Artyom Dymchenko -- VS2005 - Visual Studio 2005 project -- VS2008 - Visual Studio 2008 project -- VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) -- build - command line scripts prepared for Visual Studio compilation without IDE +- `cmake` - CMake project contributed by Artyom Dymchenko +- `VS2005` - Visual Studio 2005 project +- `VS2008` - Visual Studio 2008 project +- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) +- `build` - command line scripts prepared for Visual Studio compilation without IDE #### How to compile zstd with Visual Studio diff --git a/tests/README.md b/tests/README.md index 309638f9..72be1a9b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,24 +2,24 @@ programs and scripts for automated testing of zstd ================================ This directory contains the following programs and scripts: -- datagen : Synthetic and parametrable data generator, for tests -- fullbench : Precisely measure speed for each zstd inner functions -- fuzzer : Test tool, to check zstd integrity on target platform -- paramgrill : parameter tester for zstd -- test-zstd-speed.py : script for testing zstd speed difference between commits -- test-zstd-versions.py : compatibility test between zstd versions stored on Github (v0.1+) -- zbufftest : Test tool, to check ZBUFF integrity on target platform -- zstreamtest : Fuzzer test tool for zstd streaming API +- `datagen` : Synthetic and parametrable data generator, for tests +- `fullbench` : Precisely measure speed for each zstd inner functions +- `fuzzer` : Test tool, to check zstd integrity on target platform +- `paramgrill` : parameter tester for zstd +- `test-zstd-speed.py` : script for testing zstd speed difference between commits +- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) +- `zbufftest` : Test tool to check ZBUFF integrity on target platform +- `zstreamtest` : Fuzzer test tool for zstd streaming API -#### test-zstd-versions.py - script for testing zstd interoperability between versions +#### `test-zstd-versions.py` - script for testing zstd interoperability between versions This script creates `versionsTest` directory to which zstd repository is cloned. Then all taged (released) versions of zstd are compiled. In the following step interoperability between zstd versions is checked. -#### test-zstd-speed.py - script for testing zstd speed difference between commits +#### `test-zstd-speed.py` - script for testing zstd speed difference between commits This script creates `speedTest` directory to which zstd repository is cloned. Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter). From 637d33566dabd11527b8ccc7b6ef0ca8094a3fcc Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 10:42:49 +0200 Subject: [PATCH 37/74] minor README.md corrections --- programs/README.md | 8 ++++---- tests/README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/README.md b/programs/README.md index f38e6c49..4028756c 100644 --- a/programs/README.md +++ b/programs/README.md @@ -29,15 +29,15 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression will rely more and more on previously decoded content to compress the rest of the file. Usage of the dictionary builder and created dictionaries with CLI: -1) Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName` -2) Compress with dictionary: `zstd FILE -D dictionaryName` -3) Decompress with dictionary: `zstd --decompress FILE.zst -D dictionaryName` +1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName` +2. Compress with dictionary: `zstd FILE -D dictionaryName` +3. Decompress with dictionary: `zstd --decompress FILE.zst -D dictionaryName` #### Benchmark in Command Line Interface CLI includes in-memory compression benchmark module for zstd. -The benchmark is conducted using given filenames which are read into memory and joined together. +The benchmark is conducted using given filenames. The files are read into memory and joined together. It makes benchmark more precise as it eliminates I/O overhead. Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the `-r` option. diff --git a/tests/README.md b/tests/README.md index 72be1a9b..0d64c98d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,7 +8,7 @@ This directory contains the following programs and scripts: - `paramgrill` : parameter tester for zstd - `test-zstd-speed.py` : script for testing zstd speed difference between commits - `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) -- `zbufftest` : Test tool to check ZBUFF integrity on target platform +- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity - `zstreamtest` : Fuzzer test tool for zstd streaming API From 0132375ba6975b8acb781d83bb3f9614424ca817 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 12:20:38 +0200 Subject: [PATCH 38/74] improved zstd_compression_format.md --- programs/README.md | 1 + zstd_compression_format.md | 105 +++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/programs/README.md b/programs/README.md index 4028756c..0fbb8a35 100644 --- a/programs/README.md +++ b/programs/README.md @@ -29,6 +29,7 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression will rely more and more on previously decoded content to compress the rest of the file. Usage of the dictionary builder and created dictionaries with CLI: + 1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName` 2. Compress with dictionary: `zstd FILE -D dictionaryName` 3. Decompress with dictionary: `zstd --decompress FILE.zst -D dictionaryName` diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 61f6a57b..867a9b03 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -97,6 +97,42 @@ to decode all concatenated frames in their sequential order, delivering the final decompressed result as if it was a single content. +Skippable Frames +---------------- + +| `Magic_Number` | `Frame_Size` | `User_Data` | +|:--------------:|:------------:|:-----------:| +| 4 bytes | 4 bytes | n bytes | + +Skippable frames allow the insertion of user-defined data +into a flow of concatenated frames. +Its design is pretty straightforward, +with the sole objective to allow the decoder to quickly skip +over user-defined data and continue decoding. + +Skippable frames defined in this specification are compatible with [LZ4] ones. + +[LZ4]:http://www.lz4.org + +__`Magic_Number`__ + +4 Bytes, little-endian format. +Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. +All 16 values are valid to identify a skippable frame. + +__`Frame_Size`__ + +This is the size, in bytes, of the following `User_Data` +(without including the magic number nor the size field itself). +This field is represented using 4 Bytes, little-endian format, unsigned 32-bits. +This means `User_Data` can’t be bigger than (2^32-1) bytes. + +__`User_Data`__ + +The `User_Data` can be anything. Data will just be skipped by the decoder. + + + General Structure of Zstandard Frame format ------------------------------------------- The structure of a single Zstandard frame is following: @@ -163,9 +199,9 @@ The `Flag_Value` can be converted into `Field_Size`, which is the number of bytes used by `Frame_Content_Size` according to the following table: -|`Flag_Value`| 0 | 1 | 2 | 3 | -| ---------- | --- | --- | --- | --- | -|`Field_Size`| 0-1 | 2 | 4 | 8 | +|`Flag_Value`| 0 | 1 | 2 | 3 | +| ---------- | ------ | --- | --- | --- | +|`Field_Size`| 0 or 1 | 2 | 4 | 8 | When `Flag_Value` is `0`, `Field_Size` depends on `Single_Segment_flag` : if `Single_Segment_flag` is set, `Field_Size` is 1. @@ -361,40 +397,6 @@ up to `Block_Maximum_Decompressed_Size`, which is the smallest of : - 128 KB -Skippable Frames ----------------- - -| `Magic_Number` | `Frame_Size` | `User_Data` | -|:--------------:|:------------:|:-----------:| -| 4 bytes | 4 bytes | n bytes | - -Skippable frames allow the insertion of user-defined data -into a flow of concatenated frames. -Its design is pretty straightforward, -with the sole objective to allow the decoder to quickly skip -over user-defined data and continue decoding. - -Skippable frames defined in this specification are compatible with [LZ4] ones. - -[LZ4]:http://www.lz4.org - -__`Magic_Number`__ - -4 Bytes, little-endian format. -Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. -All 16 values are valid to identify a skippable frame. - -__`Frame_Size`__ - -This is the size, in bytes, of the following `User_Data` -(without including the magic number nor the size field itself). -This field is represented using 4 Bytes, little-endian format, unsigned 32-bits. -This means `User_Data` can’t be bigger than (2^32-1) bytes. - -__`User_Data`__ - -The `User_Data` can be anything. Data will just be skipped by the decoder. - The format of `Compressed_Block` -------------------------------- @@ -447,9 +449,12 @@ __`Literals_Block_Type`__ This field uses 2 lowest bits of first byte, describing 4 different block types : -| Value | 0 | 1 | 2 | 3 | -| --------------------- | -------------------- | -------------------- | --------------------------- | ----------------------------- | -| `Literals_Block_Type` | `Raw_Literals_Block` | `RLE_Literals_Block` | `Compressed_Literals_Block` | `Repeat_Stats_Literals_Block` | +| `Literals_Block_Type` | Value | +| ----------------------------- | ----- | +| `Raw_Literals_Block` | 0 | +| `RLE_Literals_Block` | 1 | +| `Compressed_Literals_Block` | 2 | +| `Repeat_Stats_Literals_Block` | 3 | - `Raw_Literals_Block` - Literals are stored uncompressed. - `RLE_Literals_Block` - Literals consist of a single byte value repeated N times. @@ -466,37 +471,37 @@ __`Size_Format`__ - For `Compressed_Block`, it requires to decode both `Compressed_Size` and `Regenerated_Size` (the decompressed size). It will also decode the number of streams. -- For `Raw_Block` and `RLE_Block` it's enough to decode `Regenerated_Size`. +- For `Raw_Literals_Block` and `RLE_Literals_Block` it's enough to decode `Regenerated_Size`. For values spanning several bytes, convention is little-endian. __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : -- Value : x0 : `Regenerated_Size` uses 5 bits (0-31). +- Value x0 : `Regenerated_Size` uses 5 bits (0-31). `Literals_Section_Header` has 1 byte. `Regenerated_Size = Header[0]>>3` -- Value : 01 : `Regenerated_Size` uses 12 bits (0-4095). +- Value 01 : `Regenerated_Size` uses 12 bits (0-4095). `Literals_Section_Header` has 2 bytes. `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4)` -- Value : 11 : `Regenerated_Size` uses 20 bits (0-1048575). +- Value 11 : `Regenerated_Size` uses 20 bits (0-1048575). `Literals_Section_Header` has 3 bytes. `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)` -Note : it's allowed to represent a short value (ex : `13`) -using a long format, accepting the reduced compacity. +Note : it's allowed to represent a short value (for example `13`) +using a long format, accepting the increased compressed data size. __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : -- Value : 00 : _Single stream_. +- Value 00 : _A single stream_. Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). `Literals_Section_Header` has 3 bytes. -- Value : 01 : 4 streams. +- Value 01 : 4 streams. Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). `Literals_Section_Header` has 3 bytes. -- Value : 10 : 4 streams. +- Value 10 : 4 streams. Both `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383). `Literals_Section_Header` has 4 bytes. -- Value : 11 : 4 streams. +- Value 11 : 4 streams. Both `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143). `Literals_Section_Header` has 5 bytes. From 96bdd87de4e40ae3fa921afe98a42f0184cb89fe Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 14:34:42 +0200 Subject: [PATCH 39/74] fixed : compression bug on very large files --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e0ae91bf..0a2fb9af 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2274,7 +2274,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (cctx->lowLimit > (1<<30)) { U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt); U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1; - U32 const newLowLimit = cctx->lowLimit & chainMask; /* preserve position % chainSize */ + U32 const newLowLimit = (cctx->lowLimit & chainMask) + (chainMask+1); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ U32 const correction = cctx->lowLimit - newLowLimit; ZSTD_reduceIndex(cctx, correction); cctx->base += correction; From de9d13068ffc864c9ccc75700715700b27fe520d Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 14:59:08 +0200 Subject: [PATCH 40/74] updated zstd_compression_format.md --- programs/README.md | 4 +- zstd_compression_format.md | 131 ++++++++++++++++++++----------------- 2 files changed, 72 insertions(+), 63 deletions(-) diff --git a/programs/README.md b/programs/README.md index 0fbb8a35..9bd1e71b 100644 --- a/programs/README.md +++ b/programs/README.md @@ -31,8 +31,8 @@ will rely more and more on previously decoded content to compress the rest of th Usage of the dictionary builder and created dictionaries with CLI: 1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName` -2. Compress with dictionary: `zstd FILE -D dictionaryName` -3. Decompress with dictionary: `zstd --decompress FILE.zst -D dictionaryName` +2. Compress with the dictionary: `zstd FILE -D dictionaryName` +3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName` diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 867a9b03..7143eea3 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -271,7 +271,7 @@ which can be any value from 1 to 2^64-1 bytes (16 EB). | ----------- | ---------- | ---------- | | Field name | `Exponent` | `Mantissa` | -Maximum distance is given by the following formulae : +Maximum distance is given by the following formulas : ``` windowLog = 10 + Exponent; windowBase = 1 << windowLog; @@ -415,7 +415,7 @@ To decode a compressed block, the following elements are necessary : or all previous blocks when `Single_Segment_flag` is set. - List of "recent offsets" from previous compressed block. - Decoding tables of previous compressed block for each symbol type - (literals, litLength, matchLength, offset). + (literals, literals lengths, match lengths, offsets). ### `Literals_Section` @@ -510,7 +510,7 @@ Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian conven #### `Huffman_Tree_Description` -This section is only present when `Literals_Block_Type` type is `Compressed_Block` (`2`). +This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`). Prefix coding represents symbols from an a priori known alphabet by bit sequences (codewords), one codeword for each symbol, @@ -532,9 +532,11 @@ This specification limits maximum code length to 11 bits. ##### Representation All literal values from zero (included) to last present one (excluded) -are represented by `Weight` values, from 0 to `Max_Number_of_Bits`. -Transformation from `Weight` to `Number_of_Bits` follows this formulae : -`Number_of_Bits = Weight ? Max_Number_of_Bits + 1 - Weight : 0` . +are represented by `Weight` with values from `0` to `Max_Number_of_Bits`. +Transformation from `Weight` to `Number_of_Bits` follows this formula : +``` +Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0 +``` The last symbol's `Weight` is deduced from previously decoded ones, by completing to the nearest power of 2. This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree. @@ -549,7 +551,10 @@ Let's presume the following Huffman tree must be described : The tree depth is 4, since its smallest element uses 4 bits. Value `5` will not be listed, nor will values above `5`. Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`. -Weight formula is : `Weight = Number_of_Bits ? Max_Number_of_Bits + 1 - Number_of_Bits : 0`. +Weight formula is : +``` +Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0 +``` It gives the following serie of weights : | `Weight` | 4 | 3 | 2 | 0 | 1 | @@ -580,9 +585,9 @@ which tells how to decode the list of weights. - if `headerByte` < 128 : the serie of weights is compressed by FSE. - The length of the FSE-compressed serie is `headerByte` (0-127). + The length of the FSE-compressed serie is equal to `headerByte` (0-127). -##### FSE (Finite State Entropy) compression of Huffman weights +##### Finite State Entropy (FSE) compression of Huffman weights The serie of weights is compressed using FSE compression. It's a single bitstream with 2 interleaved states, @@ -612,9 +617,10 @@ When both states have overflowed the bitstream, end is reached. ##### Conversion from weights to Huffman prefix codes All present symbols shall now have a `Weight` value. -It is possible to transform weights into Number_of_Bits, using this formula : -`Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0` . - +It is possible to transform weights into Number_of_Bits, using this formula: +``` +Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0 +``` Symbols are sorted by `Weight`. Within same `Weight`, symbols keep natural order. Symbols with a `Weight` of zero are removed. Then, starting from lowest weight, prefix codes are distributed in order. @@ -636,21 +642,21 @@ it gives the following distribution : | prefix codes | N/A | 0000| 0001| 001 | 01 | 1 | -#### Literals bitstreams +#### The content of Huffman-compressed literal stream ##### Bitstreams sizes As seen in a previous paragraph, -there are 2 flavors of Huffman-compressed literals : -single stream, and 4-streams. +there are 2 types of Huffman-compressed literals : +a single stream and 4 streams. -4-streams is useful for CPU with multiple execution units and out-of-order operations. +Encoding using 4 streams is useful for CPU with multiple execution units and out-of-order operations. Since each stream can be decoded independently, it's possible to decode them up to 4x faster than a single stream, presuming the CPU has enough parallelism available. For single stream, header provides both the compressed and regenerated size. -For 4-streams though, +For 4 streams though, header only provides compressed and regenerated size of all 4 streams combined. In order to properly decode the 4 streams, it's necessary to know the compressed and regenerated size of each stream. @@ -663,8 +669,10 @@ bitstreams are preceded by 3 unsigned little-endian 16-bits values. Each value represents the compressed size of one stream, in order. The last stream size is deducted from total compressed size and from previously decoded stream sizes : + `stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`. + ##### Bitstreams read and decode Each bitstream must be read _backward_, @@ -706,23 +714,18 @@ When all _sequences_ are decoded, if there is any literal left in the _literal section_, these bytes are added at the end of the block. -The _Sequences_Section_ regroup all symbols required to decode commands. +The `Sequences_Section` regroup all symbols required to decode commands. There are 3 symbol types : literals lengths, offsets and match lengths. They are encoded together, interleaved, in a single _bitstream_. -Each symbol is a _code_ in its own context, -which specifies a baseline and a number of bits to add. -_Codes_ are FSE compressed, -and interleaved with raw additional bits in the same bitstream. - -The Sequences section starts by a header, -followed by optional Probability tables for each symbol type, +The `Sequences_Section` starts by a header, +followed by optional probability tables for each symbol type, followed by the bitstream. | `Sequences_Section_Header` | [`Literals_Length_Table`] | [`Offset_Table`] | [`Match_Length_Table`] | bitStream | | -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- | -To decode the Sequence section, it's required to know its size. +To decode the `Sequences_Section`, it's required to know its size. This size is deducted from `blockSize - literalSectionSize`. @@ -753,8 +756,8 @@ This is a single byte, defining the compression mode of each symbol type. The last field, `Reserved`, must be all-zeroes. -`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the compression mode of -literals lengths, offsets and match lengths respectively. +`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the `Compression_Mode` of +literals lengths, offsets, and match lengths respectively. They follow the same enumeration : @@ -769,9 +772,14 @@ They follow the same enumeration : A distribution table will be present. It will be described in [next part](#distribution-tables). -#### Symbols decoding +#### The codes for literals lengths, match lengths, and offsets. -##### Literals Length codes +Each symbol is a _code_ in its own context, +which specifies `Baseline` and `Number_of_Bits` to add. +_Codes_ are FSE compressed, +and interleaved with raw additional bits in the same bitstream. + +##### Literals length codes Literals length codes are values ranging from `0` to `35` included. They define lengths from 0 to 131071 bytes. @@ -783,20 +791,20 @@ They define lengths from 0 to 131071 bytes. | `Literals_Length_Code` | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | | ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| Baseline | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 | +| `Baseline` | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 | | `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | | `Literals_Length_Code` | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | | ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| Baseline | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | +| `Baseline` | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | | `Number_of_Bits` | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | | `Literals_Length_Code` | 32 | 33 | 34 | 35 | | ---------------------- | ---- | ---- | ---- | ---- | -| Baseline | 8192 |16384 |32768 |65536 | +| `Baseline` | 8192 |16384 |32768 |65536 | | `Number_of_Bits` | 13 | 14 | 15 | 16 | -__Default distribution__ +##### Default distribution for literals length codes When `Compression_Mode` is `Predefined_Mode`, a predefined distribution is used for FSE compression. @@ -809,7 +817,7 @@ short literalsLength_defaultDistribution[36] = -1,-1,-1,-1 }; ``` -##### Match Length codes +##### Match length codes Match length codes are values ranging from `0` to `52` included. They define lengths from 3 to 131074 bytes. @@ -821,25 +829,25 @@ They define lengths from 3 to 131074 bytes. | `Match_Length_Code` | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | | ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| Baseline | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 | +| `Baseline` | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 | | `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | | `Match_Length_Code` | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | | ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| Baseline | 67 | 83 | 99 | 131 | 258 | 514 | 1026 | 2050 | +| `Baseline` | 67 | 83 | 99 | 131 | 258 | 514 | 1026 | 2050 | | `Number_of_Bits` | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | | `Match_Length_Code` | 48 | 49 | 50 | 51 | 52 | | ------------------- | ---- | ---- | ---- | ---- | ---- | -| Baseline | 4098 | 8194 |16486 |32770 |65538 | +| `Baseline` | 4098 | 8194 |16486 |32770 |65538 | | `Number_of_Bits` | 12 | 13 | 14 | 15 | 16 | -__Default distribution__ +##### Default distribution for match length codes When `Compression_Mode` is defined as `Predefined_Mode`, a predefined distribution is used for FSE compression. -Here is its definition. It uses an accuracy of 6 bits (64 states). +Below is its definition. It uses an accuracy of 6 bits (64 states). ``` short matchLengths_defaultDistribution[53] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, @@ -858,26 +866,27 @@ For information, at the time of this writing. the reference decoder supports a maximum `N` value of `28` in 64-bits mode. An offset code is also the number of additional bits to read, -and can be translated into an `Offset_Value` using the following formulae : +and can be translated into an `Offset_Value` using the following formulas : ``` Offset_Value = (1 << offsetCode) + readNBits(offsetCode); if (Offset_Value > 3) offset = Offset_Value - 3; ``` -It means that maximum `Offset_Value` is `2^(N+1))-1` and it supports back-reference distance up to 2^(N+1))-4 +It means that maximum `Offset_Value` is `2^(N+1))-1` and it supports back-reference distance up to `2^(N+1))-4` but is limited by [maximum back-reference distance](#window_descriptor). -Offset_Value from 1 to 3 are special : they define "repeat codes", +`Offset_Value` from 1 to 3 are special : they define "repeat codes", which means one of the previous offsets will be repeated. They are sorted in recency order, with 1 meaning the most recent one. See [Repeat offsets](#repeat-offsets) paragraph. -__Default distribution__ + +##### Default distribution for offset codes When `Compression_Mode` is defined as `Predefined_Mode`, a predefined distribution is used for FSE compression. -Here is its definition. It uses an accuracy of 5 bits (32 states), +Below is its definition. It uses an accuracy of 5 bits (32 states), and supports a maximum `N` of 28, allowing offset values up to 536,870,908 . If any sequence in the compressed block requires an offset larger than this, @@ -918,7 +927,7 @@ The bitstream starts by reporting on which scale it operates. Note that maximum `Accuracy_Log` for literal and match lengths is `9`, and for offsets is `8`. Higher values are considered errors. -Then follow each symbol value, from `0` to last present one. +Then follows each symbol value, from `0` to last present one. The number of bits used by each field is variable. It depends on : @@ -947,11 +956,11 @@ It depends on : Symbols probabilities are read one by one, in order. -Probability is obtained from Value decoded by following formulae : +Probability is obtained from Value decoded by following formula : `Proba = value - 1` It means value `0` becomes negative probability `-1`. -`-1` is a special probability, which means `less than 1`. +`-1` is a special probability, which means "less than 1". Its effect on distribution table is described in [next paragraph]. For the purpose of calculating cumulated distribution, it counts as one. @@ -1006,7 +1015,7 @@ typically by a "less than 1" probability symbol. The result is a list of state values. Each state will decode the current symbol. -To get the Number of bits and baseline required for next state, +To get the `Number_of_Bits` and `Baseline` required for next state, it's first necessary to sort all states in their natural order. The lower states will need 1 more bit than higher ones. @@ -1030,11 +1039,11 @@ Numbering starts from higher states using less bits. | width | 32 | 32 | 32 | 16 | 16 | | `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 | | range number | 2 | 4 | 6 | 0 | 1 | -| baseline | 32 | 64 | 96 | 0 | 16 | +| `Baseline` | 32 | 64 | 96 | 0 | 16 | | range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 | Next state is determined from current state -by reading the required number of bits, and adding the specified baseline. +by reading the required `Number_of_Bits`, and adding the specified `Baseline`. #### Bitstream @@ -1064,16 +1073,16 @@ Reminder : always keep in mind that all values are read _backward_. ##### Decoding a sequence A state gives a code. -A code provides a baseline and number of bits to add. +A code provides `Baseline` and `Number_of_Bits` to add. See [Symbol Decoding] section for details on each symbol. -Decoding starts by reading the number of bits required to decode offset. -It then does the same for match length, -and then for literals length. +Decoding starts by reading the `Number_of_Bits` required to decode `Offset`. +It then does the same for `Match_Length`, +and then for `Literals_Length`. -Offset / matchLength / litLength define a sequence. -It starts by inserting the number of literals defined by `litLength`, -then continue by copying `matchLength` bytes from `currentPos - offset`. +`Offset`, `Match_Length`, and `Literals_Length` define a sequence. +It starts by inserting the number of literals defined by `Literals_Length`, +then continue by copying `Match_Length` bytes from `currentPos - Offset`. The next operation is to update states. Using rules pre-calculated in the decoding tables, @@ -1085,7 +1094,7 @@ This operation will be repeated `Number_of_Sequences` times. At the end, the bitstream shall be entirely consumed, otherwise bitstream is considered corrupted. -[Symbol Decoding]:#symbols-decoding +[Symbol Decoding]:#the-codes-for-literals-lengths-match-lengths-and-offsets ##### Repeat offsets @@ -1143,8 +1152,8 @@ _Reserved ranges :_ __`Entropy_Tables`__ : following the same format as a [compressed blocks]. They are stored in following order : - Huffman tables for literals, FSE table for offset, - FSE table for matchLenth, and FSE table for litLength. + Huffman tables for literals, FSE table for offsets, + FSE table for match lengths, and FSE table for literals lengths. It's finally followed by 3 offset values, populating recent offsets, stored in order, 4-bytes little-endian each, for a total of 12 bytes. From 24a738c668000e18d3cd520901fe1fdb8f8c5967 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 15:15:06 +0200 Subject: [PATCH 41/74] removed TRAVIS_OS_NAME --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 75b334ff..a1893cd9 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -36,7 +36,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then MD5SUM="md5 -r" fi -$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME" +$ECHO "\nStarting playTests.sh isWindows=$isWindows" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" From 12731a9644e277e049286d7f96565b72b99ecef5 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 25 Aug 2016 15:19:37 +0200 Subject: [PATCH 42/74] minor changes in zstd_compression_format.md --- zstd_compression_format.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 7143eea3..3facb321 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -993,7 +993,7 @@ The table has a size of `tableSize = 1 << Accuracy_Log`. Each cell describes the symbol decoded, and instructions to get the next state. -Symbols are scanned in their natural order for `less than 1` probabilities. +Symbols are scanned in their natural order for "less than 1" probabilities. Symbols with this probability are being attributed a single cell, starting from the end of the table. These symbols define a full state reset, reading `Accuracy_Log` bits. @@ -1098,13 +1098,13 @@ otherwise bitstream is considered corrupted. ##### Repeat offsets -As seen in [Offset Codes], the first 3 values define a repeated offset. -They are sorted in recency order, with 1 meaning "most recent one". +As seen in [Offset Codes], the first 3 values define a repeated offset and we will call them `Repeated_Offset1`, `Repeated_Offset2`, and `Repeated_Offset3`. +They are sorted in recency order, with `Repeated_Offset1` meaning "most recent one". There is an exception though, when current sequence's literals length is `0`. -In which case, repcodes are "pushed by one", -so 1 becomes 2, 2 becomes 3, -and 3 becomes "offset_1 - 1_byte". +In which case, repeated offsets are "pushed by one", +so `Repeated_Offset1` becomes `Repeated_Offset2`, `Repeated_Offset2` becomes `Repeated_Offset3`, +and `Repeated_Offset3` becomes `Repeated_Offset1 - 1_byte`. On first block, offset history is populated by the following values : 1, 4 and 8 (in order). @@ -1119,8 +1119,8 @@ they do not contribute to offset history. New offset take the lead in offset history, up to its previous place if it was already present. -It means that when repeat offset 1 (most recent) is used, history is unmodified. -When repeat offset 2 is used, it's swapped with offset 1. +It means that when `Repeated_Offset1` (most recent) is used, history is unmodified. +When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. Dictionary format From 94376acfb0e16fc74ace873e5104a5d68e07b9e0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 19:09:21 +0200 Subject: [PATCH 43/74] fixed test on CentOS7 --- NEWS | 4 +++- tests/playTests.sh | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 70429644..6c83c9cd 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ v0.8.2 -API : ZDICT_getDictID() Small decompression speed improvement +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}() +Enhanced documentation, by Przemyslaw Skibinski +Fixed : compression fails on specific huge files, bug introduced in v0.8.1, reported by Jesse McGrew v0.8.1 New streaming API diff --git a/tests/playTests.sh b/tests/playTests.sh index a1893cd9..2f9bf045 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -196,10 +196,14 @@ mkdir dirTestDict cp *.c dirTestDict cp ../programs/*.c dirTestDict cp ../programs/*.h dirTestDict -cat dirTestDict/* | $MD5SUM > tmph1 # note : we expect same file order to generate same hash -$ZSTD -f dirTestDict/* -D tmpDictC -$ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2 -diff -q tmph1 tmph2 +$MD5SUM dirTestDict/* > tmph1 +$ZSTD -f --rm dirTestDict/* -D tmpDictC +$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum +if [[ "$OSTYPE" == "darwin"* ]]; then + $ECHO "test skipped on OS-X" # not compatible with OS-X's md5 +else + $MD5SUM -c tmph1 +fi rm -rf dirTestDict rm tmp* From bb8558164e8dd200bdaa7d6ae14ff8d643a6be78 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 19:11:11 +0200 Subject: [PATCH 44/74] fixed welcome message --- tests/zstreamtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index edfb07eb..f5f0eecb 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -668,7 +668,7 @@ int main(int argc, const char** argv) } } } /* for(argNb=1; argNb Date: Thu, 25 Aug 2016 22:22:50 +0200 Subject: [PATCH 45/74] reinforced fix for huge files --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 2f9bf045..b887613f 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -198,7 +198,7 @@ cp ../programs/*.c dirTestDict cp ../programs/*.h dirTestDict $MD5SUM dirTestDict/* > tmph1 $ZSTD -f --rm dirTestDict/* -D tmpDictC -$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum +$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default if [[ "$OSTYPE" == "darwin"* ]]; then $ECHO "test skipped on OS-X" # not compatible with OS-X's md5 else From 5a02b69215eee38ce669fc9a9907417cce38e347 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 22:24:59 +0200 Subject: [PATCH 46/74] reinforced fix for huge files --- lib/compress/zstd_compress.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0a2fb9af..64bb7d40 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2248,6 +2248,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa * Frame is supposed already started (header already produced) * @return : compressed size, or an error code */ +#include static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -2274,8 +2275,10 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (cctx->lowLimit > (1<<30)) { U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt); U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1; - U32 const newLowLimit = (cctx->lowLimit & chainMask) + (chainMask+1); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ + U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */); + U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ U32 const correction = cctx->lowLimit - newLowLimit; + fprintf(stderr, "chainMask : %u \n", chainMask); ZSTD_reduceIndex(cctx, correction); cctx->base += correction; cctx->dictBase += correction; From 0d59a6f73aed76ed478f9f381b06b0e1240aa259 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 22:42:46 +0200 Subject: [PATCH 47/74] removed debug strings --- lib/compress/zstd_compress.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 64bb7d40..2350a56e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2248,7 +2248,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa * Frame is supposed already started (header already produced) * @return : compressed size, or an error code */ -#include static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -2278,7 +2277,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */); U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ U32 const correction = cctx->lowLimit - newLowLimit; - fprintf(stderr, "chainMask : %u \n", chainMask); ZSTD_reduceIndex(cctx, correction); cctx->base += correction; cctx->dictBase += correction; From 0baa64a763a8f4e274eb037e808e309312eabce7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 Aug 2016 22:54:13 +0200 Subject: [PATCH 48/74] increased maximum memory size for 64-bits bench to 16 GB --- programs/bench.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/bench.c b/programs/bench.c index dc36d170..f773b5f0 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -58,7 +58,7 @@ #define MB *(1 <<20) #define GB *(1U<<30) -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-30)); static U32 g_compressibilityDefault = 50; From 87c18b2ebd2f62ea244dc4f78ee28560d12a4052 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 26 Aug 2016 01:43:47 +0200 Subject: [PATCH 49/74] fixed multiple minor warnings for XCode --- lib/Makefile | 2 +- lib/common/fse.h | 2 +- lib/compress/zstd_compress.c | 2 +- lib/dictBuilder/divsufsort.h | 10 ++++---- lib/legacy/zstd_v01.c | 46 ++++++++++++++++------------------ lib/legacy/zstd_v02.c | 46 ++++++++++++++++------------------ lib/legacy/zstd_v03.c | 46 ++++++++++++++++------------------ lib/legacy/zstd_v04.c | 48 +++++++++++++++++------------------- lib/legacy/zstd_v05.c | 47 +++++++++++++++++------------------ lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- lib/zstd.h | 6 ++--- programs/bench.c | 2 +- programs/zstdcli.c | 4 +-- 14 files changed, 123 insertions(+), 142 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 579a906f..2836c95d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -49,7 +49,7 @@ INCLUDEDIR=$(PREFIX)/include CPPFLAGS= -I. -I./common CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wdocumentation FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/lib/common/fse.h b/lib/common/fse.h index e711d013..720d54b1 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -503,7 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt BIT_flushBits(bitC); } -/*<===== Decompression =====>*/ +/* ====== Decompression ====== */ typedef struct { U16 tableLog; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2350a56e..dff7f204 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2599,7 +2599,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) BYTE* op = ostart; size_t fhSize = 0; - if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /*< not even init ! */ + if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { diff --git a/lib/dictBuilder/divsufsort.h b/lib/dictBuilder/divsufsort.h index dac09366..5440994a 100644 --- a/lib/dictBuilder/divsufsort.h +++ b/lib/dictBuilder/divsufsort.h @@ -36,8 +36,8 @@ extern "C" { /** * Constructs the suffix array of a given string. - * @param T[0..n-1] The input string. - * @param SA[0..n-1] The output array of suffixes. + * @param T [0..n-1] The input string. + * @param SA [0..n-1] The output array of suffixes. * @param n The length of the given string. * @param openMP enables OpenMP optimization. * @return 0 if no error occurred, -1 or -2 otherwise. @@ -47,9 +47,9 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP); /** * Constructs the burrows-wheeler transformed string of a given string. - * @param T[0..n-1] The input string. - * @param U[0..n-1] The output string. (can be T) - * @param A[0..n-1] The temporary array. (can be NULL) + * @param T [0..n-1] The input string. + * @param U [0..n-1] The output string. (can be T) + * @param A [0..n-1] The temporary array. (can be NULL) * @param n The length of the given string. * @param num_indexes The length of secondary indexes array. (can be NULL) * @param indexes The secondary indexes array. (can be NULL) diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 5540de17..58951761 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1685,7 +1685,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d /* Build DTables */ switch(LLtype) { - U32 max; case bt_rle : LLlog = 0; FSE_buildDTable_rle(DTableLL, *ip++); break; @@ -1693,17 +1692,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d LLlog = LLbits; FSE_buildDTable_raw(DTableLL, LLbits); break; default : - max = MaxLL; - headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; - ip += headerSize; - FSE_buildDTable(DTableLL, norm, max, LLlog); - } + { U32 max = MaxLL; + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; + ip += headerSize; + FSE_buildDTable(DTableLL, norm, max, LLlog); + } } switch(Offtype) { - U32 max; case bt_rle : Offlog = 0; if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */ @@ -1712,17 +1710,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d Offlog = Offbits; FSE_buildDTable_raw(DTableOffb, Offbits); break; default : - max = MaxOff; - headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); - if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; - ip += headerSize; - FSE_buildDTable(DTableOffb, norm, max, Offlog); - } + { U32 max = MaxOff; + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; + ip += headerSize; + FSE_buildDTable(DTableOffb, norm, max, Offlog); + } } switch(MLtype) { - U32 max; case bt_rle : MLlog = 0; if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */ @@ -1731,14 +1728,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d MLlog = MLbits; FSE_buildDTable_raw(DTableML, MLbits); break; default : - max = MaxML; - headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; - ip += headerSize; - FSE_buildDTable(DTableML, norm, max, MLlog); - } - } + { U32 max = MaxML; + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; + if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; + ip += headerSize; + FSE_buildDTable(DTableML, norm, max, MLlog); + } } } return ip-istart; } diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 8958eeb1..1d4abdf2 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -3100,7 +3100,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d /* Build DTables */ switch(LLtype) { - U32 max; case bt_rle : LLlog = 0; FSE_buildDTable_rle(DTableLL, *ip++); break; @@ -3108,17 +3107,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d LLlog = LLbits; FSE_buildDTable_raw(DTableLL, LLbits); break; default : - max = MaxLL; - headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (LLlog > LLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableLL, norm, max, LLlog); - } + { U32 max = MaxLL; + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (LLlog > LLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableLL, norm, max, LLlog); + } } switch(Offtype) { - U32 max; case bt_rle : Offlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3128,17 +3126,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d Offlog = Offbits; FSE_buildDTable_raw(DTableOffb, Offbits); break; default : - max = MaxOff; - headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (Offlog > OffFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableOffb, norm, max, Offlog); - } + { U32 max = MaxOff; + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (Offlog > OffFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableOffb, norm, max, Offlog); + } } switch(MLtype) { - U32 max; case bt_rle : MLlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3147,14 +3144,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d MLlog = MLbits; FSE_buildDTable_raw(DTableML, MLbits); break; default : - max = MaxML; - headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (MLlog > MLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableML, norm, max, MLlog); - } - } + { U32 max = MaxML; + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (MLlog > MLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableML, norm, max, MLlog); + } } } return ip-istart; } diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 7d5ac56f..17c43c78 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -2742,7 +2742,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d /* Build DTables */ switch(LLtype) { - U32 max; case bt_rle : LLlog = 0; FSE_buildDTable_rle(DTableLL, *ip++); break; @@ -2750,17 +2749,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d LLlog = LLbits; FSE_buildDTable_raw(DTableLL, LLbits); break; default : - max = MaxLL; - headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (LLlog > LLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableLL, norm, max, LLlog); - } + { U32 max = MaxLL; + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (LLlog > LLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableLL, norm, max, LLlog); + } } switch(Offtype) { - U32 max; case bt_rle : Offlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -2770,17 +2768,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d Offlog = Offbits; FSE_buildDTable_raw(DTableOffb, Offbits); break; default : - max = MaxOff; - headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (Offlog > OffFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableOffb, norm, max, Offlog); - } + { U32 max = MaxOff; + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (Offlog > OffFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableOffb, norm, max, Offlog); + } } switch(MLtype) { - U32 max; case bt_rle : MLlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -2789,14 +2786,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d MLlog = MLbits; FSE_buildDTable_raw(DTableML, MLbits); break; default : - max = MaxML; - headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (MLlog > MLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableML, norm, max, MLlog); - } - } + { U32 max = MaxML; + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (MLlog > MLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableML, norm, max, MLlog); + } } } return ip-istart; } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index cbf203da..da9a1dcb 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3145,7 +3145,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d /* Build DTables */ switch(LLtype) { - U32 max; case bt_rle : LLlog = 0; FSE_buildDTable_rle(DTableLL, *ip++); break; @@ -3153,17 +3152,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d LLlog = LLbits; FSE_buildDTable_raw(DTableLL, LLbits); break; default : - max = MaxLL; - headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (LLlog > LLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableLL, norm, max, LLlog); - } + { U32 max = MaxLL; + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (LLlog > LLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableLL, norm, max, LLlog); + } } switch(Offtype) { - U32 max; case bt_rle : Offlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3173,17 +3171,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d Offlog = Offbits; FSE_buildDTable_raw(DTableOffb, Offbits); break; default : - max = MaxOff; - headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (Offlog > OffFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableOffb, norm, max, Offlog); - } + { U32 max = MaxOff; + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (Offlog > OffFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableOffb, norm, max, Offlog); + } } switch(MLtype) { - U32 max; case bt_rle : MLlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3192,14 +3189,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d MLlog = MLbits; FSE_buildDTable_raw(DTableML, MLbits); break; default : - max = MaxML; - headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); - if (FSE_isError(headerSize)) return ERROR(GENERIC); - if (MLlog > MLFSELog) return ERROR(corruption_detected); - ip += headerSize; - FSE_buildDTable(DTableML, norm, max, MLlog); - } - } + { U32 max = MaxML; + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (MLlog > MLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableML, norm, max, MLlog); + } } } return ip-istart; } @@ -3701,7 +3697,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. * Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. * The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFF_isError(). * diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 7dad84d9..b64f1251 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3333,7 +3333,6 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps /* Build DTables */ switch(LLtype) { - U32 max; case FSEv05_ENCODING_RLE : LLlog = 0; FSEv05_buildDTable_rle(DTableLL, *ip++); @@ -3346,17 +3345,16 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - max = MaxLL; - headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip); - if (FSEv05_isError(headerSize)) return ERROR(GENERIC); - if (LLlog > LLFSEv05Log) return ERROR(corruption_detected); - ip += headerSize; - FSEv05_buildDTable(DTableLL, norm, max, LLlog); - } + { U32 max = MaxLL; + headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSEv05_isError(headerSize)) return ERROR(GENERIC); + if (LLlog > LLFSEv05Log) return ERROR(corruption_detected); + ip += headerSize; + FSEv05_buildDTable(DTableLL, norm, max, LLlog); + } } switch(Offtype) { - U32 max; case FSEv05_ENCODING_RLE : Offlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3370,17 +3368,16 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - max = MaxOff; - headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip); - if (FSEv05_isError(headerSize)) return ERROR(GENERIC); - if (Offlog > OffFSEv05Log) return ERROR(corruption_detected); - ip += headerSize; - FSEv05_buildDTable(DTableOffb, norm, max, Offlog); - } + { U32 max = MaxOff; + headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSEv05_isError(headerSize)) return ERROR(GENERIC); + if (Offlog > OffFSEv05Log) return ERROR(corruption_detected); + ip += headerSize; + FSEv05_buildDTable(DTableOffb, norm, max, Offlog); + } } switch(MLtype) { - U32 max; case FSEv05_ENCODING_RLE : MLlog = 0; if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ @@ -3394,13 +3391,13 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - max = MaxML; - headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip); - if (FSEv05_isError(headerSize)) return ERROR(GENERIC); - if (MLlog > MLFSEv05Log) return ERROR(corruption_detected); - ip += headerSize; - FSEv05_buildDTable(DTableML, norm, max, MLlog); - } } + { U32 max = MaxML; + headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSEv05_isError(headerSize)) return ERROR(GENERIC); + if (MLlog > MLFSEv05Log) return ERROR(corruption_detected); + ip += headerSize; + FSEv05_buildDTable(DTableML, norm, max, MLlog); + } } } return ip-istart; } @@ -4030,7 +4027,7 @@ static size_t ZBUFFv05_limitCopy(void* dst, size_t maxDstSize, const void* src, * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. * Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. * The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFFv05_isError(). * diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index ae10d153..bc440c2c 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1383,7 +1383,7 @@ static unsigned char FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_ *******************************************/ -/*<===== Decompression =====>*/ +/* ====== Decompression ====== */ typedef struct { U16 tableLog; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 591a3c15..c4ccd24c 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1147,7 +1147,7 @@ static unsigned char FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_ /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ -/*<===== Decompression =====>*/ +/* ====== Decompression ====== */ typedef struct { U16 tableLog; diff --git a/lib/zstd.h b/lib/zstd.h index fd6170c8..706bc09b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -116,7 +116,7 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readab * Explicit memory management ***************************************/ /** Compression context */ -typedef struct ZSTD_CCtx_s ZSTD_CCtx; /*< incomplete type */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); @@ -125,7 +125,7 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); /** Decompression context */ -typedef struct ZSTD_DCtx_s ZSTD_DCtx; /*< incomplete type */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); @@ -328,7 +328,7 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f /*--- Types ---*/ -typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ +typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ diff --git a/programs/bench.c b/programs/bench.c index f773b5f0..dc36d170 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -58,7 +58,7 @@ #define MB *(1 <<20) #define GB *(1U<<30) -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-30)); +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); static U32 g_compressibilityDefault = 50; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 69188d16..c03d6dde 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -478,8 +478,8 @@ int main(int argCount, char** argv) } /* No input filename ==> use stdin and stdout */ - filenameIdx += !filenameIdx; /*< default input is stdin */ - if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */ + filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */ + if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /* when input is stdin, default output is stdout */ /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); From cb5a3207058b80ba6d4d7118230cbf95ac323846 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 26 Aug 2016 08:06:36 +0200 Subject: [PATCH 50/74] made -Wdocumentation a clang only flag --- Makefile | 2 +- lib/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a6b2fb42..b7e0649c 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ gcc6test: clean clangtest: clean clang -v - $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" + $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" armtest: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster diff --git a/lib/Makefile b/lib/Makefile index 2836c95d..579a906f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -49,7 +49,7 @@ INCLUDEDIR=$(PREFIX)/include CPPFLAGS= -I. -I./common CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wdocumentation + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) From 9a021c1aaebac28d3f0756c058cdaa33cb8942d1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 26 Aug 2016 09:05:06 +0200 Subject: [PATCH 51/74] fixed some minor clang warnings --- lib/legacy/zstd_v05.c | 12 ++++++------ tests/zbufftest.c | 2 +- tests/zstreamtest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index b64f1251..2a08538e 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -859,16 +859,16 @@ void FSEv05_freeDTable(FSEv05_DTable* dt); /*! FSEv05_buildDTable(): Builds 'dt', which must be already allocated, using FSEv05_createDTable() - return : 0, - or an errorCode, which can be tested using FSEv05_isError() */ + @return : 0, + or an errorCode, which can be tested using FSEv05_isError() */ size_t FSEv05_buildDTable (FSEv05_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); /*! FSEv05_decompress_usingDTable(): - Decompress compressed source @cSrc of size @cSrcSize using @dt - into @dst which must be already allocated. - return : size of regenerated data (necessarily <= @dstCapacity) - or an errorCode, which can be tested using FSEv05_isError() */ + Decompress compressed source @cSrc of size @cSrcSize using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= @dstCapacity) + or an errorCode, which can be tested using FSEv05_isError() */ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv05_DTable* dt); diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 151785c6..563a5c9a 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -183,7 +183,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo cSize += genSize; genSize = compressedBufferSize - cSize; { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); - if (r != 0) goto _output_error; } /*< error, or some data not flushed */ + if (r != 0) goto _output_error; } /* error, or some data not flushed */ cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f5f0eecb..dd00864d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -185,7 +185,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (ZSTD_isError(r)) goto _output_error; } if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /*< error, or some data not flushed */ + if (r != 0) goto _output_error; } /* error, or some data not flushed */ cSize += outBuff.pos; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); From e19a9ef05d797cd3f72b34de30fac8f506c9415d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 26 Aug 2016 20:02:49 +0200 Subject: [PATCH 52/74] update compression level table --- lib/compress/zstd_compress.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index dff7f204..1761a448 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3056,15 +3056,15 @@ int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, TL, strat */ - { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */ + { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ - { 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ + { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ - { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8.*/ + { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ @@ -3073,16 +3073,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ - { 23, 23, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 17.*/ - { 23, 23, 22, 6, 5, 24, ZSTD_btopt }, /* level 18.*/ - { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19.*/ - { 25, 26, 23, 7, 3, 64, ZSTD_btopt }, /* level 20.*/ - { 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21.*/ - { 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22.*/ + { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ + { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ + { 25, 25, 23, 7, 3, 64, ZSTD_btopt }, /* level 20 */ + { 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21 */ + { 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22 */ }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ - { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ From bbd5bb75a647bad95fda017299e75aef63f2515e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 27 Aug 2016 11:41:23 +0200 Subject: [PATCH 53/74] update table --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f8353ec1..bdea53c9 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ As a reference, several fast compression algorithms were tested and compared on |Name | Ratio | C.speed | D.speed | |-----------------|-------|--------:|--------:| | | | MB/s | MB/s | -|**zstd 0.8.0 -1**|**2.877**|**330**| **930** | +|**zstd 0.8.2 -1**|**2.877**|**330**| **940** | | [zlib] 1.2.8 -1 | 2.730 | 95 | 360 | | brotli 0.4 -0 | 2.708 | 320 | 375 | | QuickLZ 1.5 | 2.237 | 510 | 605 | From 4bf317dd0099bd11690ebee4d72f7ac6919e5964 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 07:43:34 -0700 Subject: [PATCH 54/74] first version supporting legacy streams (transparent decoding) --- lib/common/error_private.h | 1 + lib/common/error_public.h | 3 +- lib/decompress/zstd_decompress.c | 50 ++++++++- lib/legacy/zstd_legacy.h | 186 +++++++++++++++++++++++++++---- lib/zstd.h | 6 +- 5 files changed, 215 insertions(+), 31 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 34c0c4c0..977fe3d4 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 10f020cf..5a6d7107 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -41,13 +41,14 @@ extern "C" { #include /* size_t */ -/* **************************************** +/*-**************************************** * error codes list ******************************************/ typedef enum { ZSTD_error_no_error, ZSTD_error_GENERIC, ZSTD_error_prefix_unknown, + ZSTD_error_version_unsupported, ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b96f487a..5b95de96 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1308,8 +1308,8 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); + dst, dstCapacity, + src, srcSize); } @@ -1337,6 +1337,11 @@ struct ZSTD_DStream_s { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; ZSTD_customMem customMem; + void* dictContent; + size_t dictSize; + const void* dictSource; + void* legacyContext; + U32 legacyVersion; }; /* typedef'd to ZSTD_DStream within "zstd.h" */ @@ -1372,6 +1377,13 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) ZSTD_freeDCtx(zds->zd); if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (zds->legacyContext) { + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->legacyVersion); + zds->legacyContext = NULL; + } +#endif zds->customMem.customFree(zds->customMem.opaque, zds); return 0; } @@ -1386,7 +1398,18 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize); + if ((dict != zds->dictSource) || (dictSize != zds->dictSize)) { /* new dictionary */ + if (dictSize > zds->dictSize) { + if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); + zds->dictContent = zds->customMem.customAlloc(zds->customMem.opaque, dictSize); + if (zds->dictContent == NULL) return ERROR(memory_allocation); + } + memcpy(zds->dictContent, dict, dictSize); + zds->dictSize = dictSize; + } + if (zds->legacyContext) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); /* legacy restarts from scratch on each call, to detect restart */ + zds->legacyVersion = 0; + return 0; } size_t ZSTD_initDStream(ZSTD_DStream* zds) @@ -1432,6 +1455,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB char* op = ostart; U32 someMoreWork = 1; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) + return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); +#endif + while (someMoreWork) { switch(zds->stage) { @@ -1439,8 +1467,19 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return ERROR(init_missing); case zdss_loadHeader : - { size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize); - if (ZSTD_isError(hSize)) return hSize; + { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart) + if (legacyVersion) { + zds->legacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); + } else { + return hSize; /* error */ + } } +#else + return hSize; +#endif if (hSize != 0) { /* need more input */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ @@ -1454,6 +1493,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ + ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 6c2a1019..a5a767e4 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -42,6 +42,7 @@ extern "C" { ***************************************/ #include "mem.h" /* MEM_STATIC */ #include "error_private.h" /* ERROR */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer */ #include "zstd_v01.h" #include "zstd_v02.h" #include "zstd_v03.h" @@ -76,32 +77,30 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) { - if (srcSize < 4) return 0; - - { U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } - return 0; /* should not be possible */ + U32 const version = ZSTD_isLegacy(src, srcSize); + if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; } + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + if (version==7) { + ZSTDv07_frameParams fParams; + size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + return 0; /* should not be possible */ } + MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, @@ -148,6 +147,147 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } +MEM_STATIC void* ZSTD_createLegacyStreamContext(U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return NULL; + case 4 : return ZBUFFv04_createDCtx(); + case 5 : return ZBUFFv05_createDCtx(); + case 6 : return ZBUFFv06_createDCtx(); + case 7 : return ZBUFFv07_createDCtx(); + } +} + +MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return ERROR(version_unsupported); + case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); + case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); + case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); + case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); + } +} + + +MEM_STATIC void ZSTD_initLegacyStream(void* legacyContext, U32 version, + const void* dict, size_t dictSize) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return; + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; + ZBUFFv04_decompressInit(dctx); + ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); + return; + } + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; + ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); + return; + } + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; + ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); + return; + } + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; + ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); + return; + } + } +} + + + +MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, + ZSTD_outBuffer* output, ZSTD_inBuffer* input, + const void* dict, size_t dictSize) +{ + if (*legacyContext == NULL) { + *legacyContext = ZSTD_createLegacyStreamContext(version); + if (*legacyContext==NULL) return ERROR(memory_allocation); + ZSTD_initLegacyStream(*legacyContext, version, dict, dictSize); + } + + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return ERROR(version_unsupported); + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + } +} + #if defined (__cplusplus) } diff --git a/lib/zstd.h b/lib/zstd.h index 706bc09b..1f3dddb2 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -208,7 +208,7 @@ typedef struct ZSTD_outBuffer_s { } ZSTD_outBuffer; -/*====== compression ======*/ +/*====== streaming compression ======*/ /*-*********************************************************************** * Streaming compression - howto @@ -225,7 +225,9 @@ typedef struct ZSTD_outBuffer_s { * The function will automatically update both `pos`. * Note that it may not consume the entire input, in which case `pos < size`, * and it's up to the caller to present again remaining data. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* @return : a size hint, preferred nb of bytes to use as input for next function call +* (it's just a hint, to help latency a little, any other value will work fine) +* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() ) * or an error code, which can be tested using ZSTD_isError(). * * At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). From 767d8f66fa115d93729badc7f83edc40eb29c6bd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 08:19:47 -0700 Subject: [PATCH 55/74] legacy contexts can be re-used --- lib/decompress/zstd_decompress.c | 14 ++++--- lib/legacy/zstd_legacy.h | 68 +++++++++++++------------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5b95de96..97adec20 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1341,6 +1341,7 @@ struct ZSTD_DStream_s { size_t dictSize; const void* dictSource; void* legacyContext; + U32 previousLegacyVersion; U32 legacyVersion; }; /* typedef'd to ZSTD_DStream within "zstd.h" */ @@ -1380,7 +1381,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (zds->legacyContext) { - ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->legacyVersion); + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); zds->legacyContext = NULL; } #endif @@ -1407,7 +1408,6 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di memcpy(zds->dictContent, dict, dictSize); zds->dictSize = dictSize; } - if (zds->legacyContext) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); /* legacy restarts from scratch on each call, to detect restart */ zds->legacyVersion = 0; return 0; } @@ -1470,10 +1470,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); if (ZSTD_isError(hSize)) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) - { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart) + { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - zds->legacyVersion = legacyVersion; - return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); + size_t initResult; + initResult = ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, + zds->dictContent, zds->dictSize); + if (ZSTD_isError(initResult)) return initResult; + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); } else { return hSize; /* error */ } } diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index a5a767e4..58985233 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -147,22 +147,6 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } -MEM_STATIC void* ZSTD_createLegacyStreamContext(U32 version) -{ - switch(version) - { - default : - case 1 : - case 2 : - case 3 : - return NULL; - case 4 : return ZBUFFv04_createDCtx(); - case 5 : return ZBUFFv05_createDCtx(); - case 6 : return ZBUFFv06_createDCtx(); - case 7 : return ZBUFFv07_createDCtx(); - } -} - MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) { switch(version) @@ -180,56 +164,58 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) } -MEM_STATIC void ZSTD_initLegacyStream(void* legacyContext, U32 version, - const void* dict, size_t dictSize) +MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, + const void* dict, size_t dictSize) { - switch(version) + if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); + switch(newVersion) { default : case 1 : case 2 : case 3 : - return; + return 0; case 4 : { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; + ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); ZBUFFv04_decompressInit(dctx); ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); - return; + *legacyContext = dctx; + return 0; } case 5 : { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; + ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); - return; + *legacyContext = dctx; + return 0; } case 6 : { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; + ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); - return; + *legacyContext = dctx; + return 0; } case 7 : { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; + ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); - return; + *legacyContext = dctx; + return 0; } } } -MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, - ZSTD_outBuffer* output, ZSTD_inBuffer* input, - const void* dict, size_t dictSize) +MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, + ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - if (*legacyContext == NULL) { - *legacyContext = ZSTD_createLegacyStreamContext(version); - if (*legacyContext==NULL) return ERROR(memory_allocation); - ZSTD_initLegacyStream(*legacyContext, version, dict, dictSize); - } - switch(version) { default : @@ -239,7 +225,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, return ERROR(version_unsupported); case 4 : { - ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) (*legacyContext); + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; const void* src = (const char*)input->src + input->pos; size_t readSize = input->size - input->pos; void* dst = (char*)output->dst + output->pos; @@ -251,7 +237,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, } case 5 : { - ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) (*legacyContext); + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; const void* src = (const char*)input->src + input->pos; size_t readSize = input->size - input->pos; void* dst = (char*)output->dst + output->pos; @@ -263,7 +249,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, } case 6 : { - ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) (*legacyContext); + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; const void* src = (const char*)input->src + input->pos; size_t readSize = input->size - input->pos; void* dst = (char*)output->dst + output->pos; @@ -275,7 +261,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, } case 7 : { - ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) (*legacyContext); + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; const void* src = (const char*)input->src + input->pos; size_t readSize = input->size - input->pos; void* dst = (char*)output->dst + output->pos; From ab267e772c3fc44cc180fe527487b5403bceeab5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 08:46:25 -0700 Subject: [PATCH 56/74] removed fileio specific legacy support; now depends on ZSTD_decompressStream() transparent legacy support --- programs/fileio.c | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c10c2896..93ecaec3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -27,15 +27,6 @@ The license of this file is GPLv2. */ -/* ************************************* -* Tuning options -***************************************/ -#ifndef ZSTD_LEGACY_SUPPORT -/* LEGACY_SUPPORT : -* decompressor can decode older formats (starting from Zstd 0.1+) */ -# define ZSTD_LEGACY_SUPPORT 1 -#endif - /* ************************************* * Compiler Options @@ -45,7 +36,7 @@ # pragma warning(disable : 4204) /* non-constant aggregate initializer */ #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif /*-************************************* @@ -63,11 +54,6 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) -# include "zstd_legacy.h" /* ZSTD_isLegacy */ -# include "fileio_legacy.h" /* FIO_decompressLegacyFrame */ -#endif - /*-************************************* * OS-specific Includes @@ -702,12 +688,6 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) readSomething = 1; if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ { U32 const magic = MEM_readLE32(ress.srcBuffer); -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) - if (ZSTD_isLegacy(ress.srcBuffer, 4)) { - filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic); - continue; - } -#endif if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)) { if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); From 5f53b0335ef1b1925fb10e94efea938c63f44063 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 10:00:49 -0700 Subject: [PATCH 57/74] fixed continuation context --- lib/decompress/zstd_decompress.c | 4 ++-- lib/legacy/zstd_v07.c | 3 ++- programs/fileio.c | 20 +++++++++++++++++++- programs/zstdcli.c | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 97adec20..e540e3fe 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1399,7 +1399,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - if ((dict != zds->dictSource) || (dictSize != zds->dictSize)) { /* new dictionary */ + if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ if (dictSize > zds->dictSize) { if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); zds->dictContent = zds->customMem.customAlloc(zds->customMem.opaque, dictSize); @@ -1457,7 +1457,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) if (zds->legacyVersion) - return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); #endif while (someMoreWork) { diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index c4ccd24c..16c09af4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4643,9 +4643,9 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, case ZBUFFds_loadHeader : { size_t const hSize = ZSTDv07_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (ZSTDv07_isError(hSize)) return hSize; if (hSize != 0) { size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ - if (ZSTDv07_isError(hSize)) return hSize; if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); zbd->lhSize += iend-ip; @@ -4685,6 +4685,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; + /* pass-through */ case ZBUFFds_read: { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); diff --git a/programs/fileio.c b/programs/fileio.c index 93ecaec3..00cb31ba 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -28,6 +28,16 @@ */ +/* ************************************* + * Tuning options + ***************************************/ +#ifndef ZSTD_LEGACY_SUPPORT +/* LEGACY_SUPPORT : + * decompressor can decode older formats (starting from Zstd 0.1+) */ +# define ZSTD_LEGACY_SUPPORT 1 +#endif + + /* ************************************* * Compiler Options ***************************************/ @@ -54,6 +64,10 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) +# include "zstd_legacy.h" /* ZSTD_isLegacy */ +#endif + /*-************************************* * OS-specific Includes @@ -688,7 +702,11 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) readSomething = 1; if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ { U32 const magic = MEM_readLE32(ress.srcBuffer); - if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)) { + if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + & (!ZSTD_isLegacy(ress.srcBuffer, toRead)) +#endif + ) { if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */ unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c03d6dde..64d2c2a9 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1,3 +1,4 @@ + /* zstdcli - Command Line Interface (cli) for zstd Copyright (C) Yann Collet 2014-2016 From 4b7b9fca320b9382e7c72595ce15dbb04fb33ccc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 10:14:27 -0700 Subject: [PATCH 58/74] better clean --- tests/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Makefile b/tests/Makefile index c054bbbb..6b22e6db 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -143,6 +143,7 @@ clean: $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ + zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) @echo Cleaning completed From 1c69baa1a9fc1b1343c7644da7b46ef8300ba45a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 12:47:17 -0700 Subject: [PATCH 59/74] CLI supports Legacy formats v0.4+ --- programs/Makefile | 4 ++-- programs/fileio.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index dd6e7838..083b0f2f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -86,10 +86,10 @@ all: zstd $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@ + $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ $(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@ + $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c diff --git a/programs/fileio.c b/programs/fileio.c index 00cb31ba..55ba1425 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -610,7 +610,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) unsigned long long FIO_decompressFrame(dRess_t ress, FILE* foutput, FILE* finput, size_t alreadyLoaded) { - U64 frameSize = 0; + U64 frameSize = 0; size_t readSize; U32 storedSkips = 0; @@ -721,7 +721,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize); + DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); /* Close */ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ From 9097f7b1562081f7adc6653670885eb795904fc8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 16:25:56 -0700 Subject: [PATCH 60/74] fixed versionsTest --- programs/legacy/fileio_legacy.c | 9 ++++----- tests/test-zstd-versions.py | 6 ++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/programs/legacy/fileio_legacy.c b/programs/legacy/fileio_legacy.c index c07b6e59..69d6db55 100644 --- a/programs/legacy/fileio_legacy.c +++ b/programs/legacy/fileio_legacy.c @@ -155,7 +155,6 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput) BYTE* const oend = outBuff + outBuffSize; U64 filesize = 0; size_t toRead; - size_t sizeCheck; ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx(); @@ -164,8 +163,8 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput) /* restore header, already read from input */ MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE); - sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */ - if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header"); + { size_t const sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */ + if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header"); } /* Main decompression Loop */ toRead = ZSTDv01_nextSrcSizeToDecompress(dctx); @@ -185,8 +184,8 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput) if (decodedSize) { /* not a header */ /* Write block */ - sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file"); + { size_t const sizeCheck = fwrite(op, 1, decodedSize, foutput); + if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file"); } filesize += decodedSize; op += decodedSize; if (op==oend) op = outBuff; diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 6cea8546..4ee778fe 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Test zstd interoperability between versions""" -# Based on LZ4 version test script, by Takayuki Matsuoka +# Copyright Yann Collet, Przemyslaw Skibinski and Takayuki Matsuoka +# License GPLv2 import filecmp import glob @@ -152,6 +153,7 @@ def decompress_zst(tag): print('OK ') else: print('command does not work') + dec_error = 1 return dec_error @@ -261,6 +263,6 @@ if __name__ == '__main__': print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) if error_code != 0: - print('==== ERROR !!! =====') + print('====== ERROR !!! =======') sys.exit(error_code) From dea67a66ab346294dad9494cc255de10a5ed3ca3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 16:56:17 -0700 Subject: [PATCH 61/74] backward compatibility versions tests start from v0.4.0 --- tests/test-zstd-versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 4ee778fe..a0a179ce 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -205,7 +205,8 @@ if __name__ == '__main__': # Retrieve all release tags print('Retrieve all release tags :') os.chdir(clone_dir) - tags = get_git_tags() + [head] + alltags = get_git_tags() + [head] + tags = [t for t in alltags if t >= 'v0.4.0'] print(tags) # Build all release zstd From 54caf6893e47724c4e5e0b1ba039c20b6fc85f8e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 19:20:13 -0700 Subject: [PATCH 62/74] Removed fileio_legacy; now legacy supports relies entirely on streaming API. CLI legacy support now limited to v0.4+ --- NEWS | 1 + programs/Makefile | 4 +- programs/legacy/fileio_legacy.c | 672 ------------------------- programs/legacy/fileio_legacy.h | 48 -- projects/cmake/programs/CMakeLists.txt | 3 +- 5 files changed, 4 insertions(+), 724 deletions(-) delete mode 100644 programs/legacy/fileio_legacy.c delete mode 100644 programs/legacy/fileio_legacy.h diff --git a/NEWS b/NEWS index 6c83c9cd..b8f8cca0 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v0.8.2 Small decompression speed improvement API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}() +CLI Legacy support for formats v0.4+ Enhanced documentation, by Przemyslaw Skibinski Fixed : compression fails on specific huge files, bug introduced in v0.8.1, reported by Jesse McGrew diff --git a/programs/Makefile b/programs/Makefile index 083b0f2f..89859416 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -63,8 +63,8 @@ CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 ZSTDLEGACY_FILES:= else ZSTD_LEGACY_SUPPORT:=1 -CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy -ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c +CPPFLAGS += -I$(ZSTDDIR)/legacy +ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c endif diff --git a/programs/legacy/fileio_legacy.c b/programs/legacy/fileio_legacy.c deleted file mode 100644 index 69d6db55..00000000 --- a/programs/legacy/fileio_legacy.c +++ /dev/null @@ -1,672 +0,0 @@ -/* - fileio_legacy.c - File i/o handler for legacy format - Copyright (C) Yann Collet 2015-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net - - zstd source repository : https://github.com/Cyan4973/zstd -*/ -/* - Note : this file is not part of ZSTD compression library. - The license of ZSTD library is BSD. - The license of this file is GPLv2. -*/ - -/* ************************************* -* Compiler Options -***************************************/ -/* Disable some Visual warning messages */ -#ifdef _MSC_VER -# define _CRT_SECURE_NO_WARNINGS -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ -#define _POSIX_SOURCE 1 /* enable fileno() within on unix */ - - -/* ************************************* -* Includes -***************************************/ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ -#include /* malloc, free */ -#include /* strcmp, strlen */ -#include /* clock */ -#include /* errno */ -#include "mem.h" -#include "fileio_legacy.h" -#include "zstd_legacy.h" /* legacy support */ - - -/* ************************************* -* OS-specific Includes -***************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# ifdef __MINGW32__ - /* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into */ -# endif -# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# include /* isatty */ -# define SET_BINARY_MODE(file) -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#endif - - -/* ************************************* -* Constants -***************************************/ -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _6BITS 0x3F -#define _8BITS 0xFF - -#define BIT6 0x40 -#define BIT7 0x80 - -#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ -#define FSE_CHECKSUM_SEED 0 - -#define CACHELINE 64 - - -/* ************************************* -* Macros -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 1; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((FIO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const unsigned refreshRate = 150; -static clock_t g_time = 0; - - -/* ************************************* -* Local Parameters -***************************************/ -void FIO_legacy_setNotificationLevel(unsigned level) { g_displayLevel=level; } - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ************************************* -* Functions -***************************************/ -static unsigned FIO_GetMilliSpan(clock_t nPrevious) -{ - clock_t nCurrent = clock(); - unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC); - return nSpan; -} - - -unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput) -{ - size_t const outBuffSize = 512 KB; - BYTE* outBuff = (BYTE*)malloc(outBuffSize); - size_t inBuffSize = 128 KB + 8; - BYTE inBuff[128 KB + 8]; - BYTE* op = outBuff; - BYTE* const oend = outBuff + outBuffSize; - U64 filesize = 0; - size_t toRead; - ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx(); - - - /* init */ - if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame"); - - /* restore header, already read from input */ - MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE); - { size_t const sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */ - if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header"); } - - /* Main decompression Loop */ - toRead = ZSTDv01_nextSrcSizeToDecompress(dctx); - while (toRead){ - size_t readSize, decodedSize; - - /* Fill input buffer */ - if (toRead > inBuffSize) - EXM_THROW(43, "too large block"); - readSize = fread(inBuff, 1, toRead, finput); - if (readSize != toRead) - EXM_THROW(44, "Read error"); - - /* Decode block */ - decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize); - if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted"); - - if (decodedSize) { /* not a header */ - /* Write block */ - { size_t const sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file"); } - filesize += decodedSize; - op += decodedSize; - if (op==oend) op = outBuff; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); - } - - /* prepare for next Block */ - toRead = ZSTDv01_nextSrcSizeToDecompress(dctx); - } - - /* release resources */ - free(outBuff); - free(dctx); - return filesize; -} - - -unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput) -{ - size_t const outBuffSize = 512 KB; - BYTE* outBuff = (BYTE*)malloc(outBuffSize); - size_t inBuffSize = 128 KB + 8; - BYTE inBuff[128 KB + 8]; - BYTE* op = outBuff; - BYTE* const oend = outBuff + outBuffSize; - U64 filesize = 0; - size_t toRead; - size_t sizeCheck; - ZSTDv02_Dctx* dctx = ZSTDv02_createDCtx(); - - /* init */ - if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame"); - - /* restore header, already read from input */ - MEM_writeLE32(inBuff, ZSTDv02_magicNumber); - sizeCheck = ZSTDv02_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv02_magicNumber)); /* Decode frame header */ - if (ZSTDv02_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header"); - - /* Main decompression Loop */ - toRead = ZSTDv02_nextSrcSizeToDecompress(dctx); - while (toRead) { - size_t readSize, decodedSize; - - /* Fill input buffer */ - if (toRead > inBuffSize) - EXM_THROW(43, "too large block"); - readSize = fread(inBuff, 1, toRead, finput); - if (readSize != toRead) - EXM_THROW(44, "Read error"); - - /* Decode block */ - decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize); - if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted"); - - if (decodedSize) { /* not a header */ - /* Write block */ - sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file"); - filesize += decodedSize; - op += decodedSize; - if (op==oend) op = outBuff; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); - } - - /* prepare for next Block */ - toRead = ZSTDv02_nextSrcSizeToDecompress(dctx); - } - - /* release resources */ - free(outBuff); - free(dctx); - return filesize; -} - - -unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput) -{ - size_t const outBuffSize = 512 KB; - BYTE* outBuff = (BYTE*)malloc(outBuffSize); - size_t inBuffSize = 128 KB + 8; - BYTE inBuff[128 KB + 8]; - BYTE* op = outBuff; - BYTE* const oend = outBuff + outBuffSize; - U64 filesize = 0; - size_t toRead; - size_t sizeCheck; - ZSTDv03_Dctx* dctx = ZSTDv03_createDCtx(); - - /* init */ - if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame"); - - /* restore header, already read from input */ - MEM_writeLE32(inBuff, ZSTDv03_magicNumber); - sizeCheck = ZSTDv03_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv03_magicNumber)); /* Decode frame header */ - if (ZSTDv03_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header"); - - /* Main decompression Loop */ - toRead = ZSTDv03_nextSrcSizeToDecompress(dctx); - while (toRead) { - size_t readSize, decodedSize; - - /* Fill input buffer */ - if (toRead > inBuffSize) - EXM_THROW(43, "too large block"); - readSize = fread(inBuff, 1, toRead, finput); - if (readSize != toRead) - EXM_THROW(44, "Read error"); - - /* Decode block */ - decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize); - if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted"); - - if (decodedSize) { /* not a header */ - /* Write block */ - sizeCheck = fwrite(op, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file"); - filesize += decodedSize; - op += decodedSize; - if (op==oend) op = outBuff; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) ); - } - - /* prepare for next Block */ - toRead = ZSTDv03_nextSrcSizeToDecompress(dctx); - } - - /* release resources */ - free(outBuff); - free(dctx); - return filesize; -} - - -/*===== v0.4.x =====*/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - void* dictBuffer; - size_t dictBufferSize; - ZBUFFv04_DCtx* dctx; -} dRessv04_t; - -static dRessv04_t FIOv04_createDResources(void) -{ - dRessv04_t ress; - - /* init */ - ress.dctx = ZBUFFv04_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.dictBuffer = NULL; ress.dictBufferSize=0; - - /* Allocate Memory */ - ress.srcBufferSize = ZBUFFv04_recommendedDInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFFv04_recommendedDOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - return ress; -} - -static void FIOv04_freeDResources(dRessv04_t ress) -{ - size_t errorCode = ZBUFFv04_freeDCtx(ress.dctx); - if (ZBUFFv04_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv04_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); - free(ress.dictBuffer); -} - - -unsigned long long FIOv04_decompressFrame(dRessv04_t ress, - FILE* foutput, FILE* finput) -{ - U64 frameSize = 0; - size_t readSize = 4; - - MEM_writeLE32(ress.srcBuffer, ZSTDv04_magicNumber); - ZBUFFv04_decompressInit(ress.dctx); - ZBUFFv04_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); - - while (1) { - /* Decode */ - size_t sizeCheck; - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t toRead = ZBUFFv04_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFFv04_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv04_getErrorName(toRead)); - readSize -= inSize; - - /* Write block */ - sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); - frameSize += decodedSize; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); - - if (toRead == 0) break; - if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); - - /* Fill input buffer */ - if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); - readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); - } - - return frameSize; -} - - -/*===== v0.5.x =====*/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - const void* dictBuffer; - size_t dictBufferSize; - ZBUFFv05_DCtx* dctx; -} dRessv05_t; - -static dRessv05_t FIOv05_createDResources(void) -{ - dRessv05_t ress; - - /* init */ - ress.dctx = ZBUFFv05_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.dictBuffer = NULL; ress.dictBufferSize=0; - - /* Allocate Memory */ - ress.srcBufferSize = ZBUFFv05_recommendedDInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFFv05_recommendedDOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - return ress; -} - -static void FIOv05_freeDResources(dRessv05_t ress) -{ - size_t const errorCode = ZBUFFv05_freeDCtx(ress.dctx); - if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); -} - - -unsigned long long FIOv05_decompressFrame(dRessv05_t ress, - FILE* foutput, FILE* finput) -{ - U64 frameSize = 0; - size_t readSize = 4; - - MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER); - ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); - - while (1) { - /* Decode */ - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead)); - readSize -= inSize; - - /* Write block */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); } - frameSize += decodedSize; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); - - if (toRead == 0) break; - if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); - - /* Fill input buffer */ - if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); - readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); - } - - return frameSize; -} - - -/*===== v0.6.x =====*/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - const void* dictBuffer; - size_t dictBufferSize; - ZBUFFv06_DCtx* dctx; -} dRessv06_t; - -static dRessv06_t FIOv06_createDResources(void) -{ - dRessv06_t ress; - - /* init */ - ress.dctx = ZBUFFv06_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.dictBuffer = NULL; ress.dictBufferSize=0; - - /* Allocate Memory */ - ress.srcBufferSize = ZBUFFv06_recommendedDInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFFv06_recommendedDOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - return ress; -} - -static void FIOv06_freeDResources(dRessv06_t ress) -{ - size_t const errorCode = ZBUFFv06_freeDCtx(ress.dctx); - if (ZBUFFv06_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv06_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); -} - - -unsigned long long FIOv06_decompressFrame(dRessv06_t ress, - FILE* foutput, FILE* finput) -{ - U64 frameSize = 0; - size_t readSize = 4; - - MEM_writeLE32(ress.srcBuffer, ZSTDv06_MAGICNUMBER); - ZBUFFv06_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); - - while (1) { - /* Decode */ - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t toRead = ZBUFFv06_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFFv06_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv06_getErrorName(toRead)); - readSize -= inSize; - - /* Write block */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); } - frameSize += decodedSize; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); - - if (toRead == 0) break; - if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); - - /* Fill input buffer */ - if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); - readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); - } - - return frameSize; -} - - -/*===== v0.7.x =====*/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - const void* dictBuffer; - size_t dictBufferSize; - ZBUFFv07_DCtx* dctx; -} dRessv07_t; - -static dRessv07_t FIOv07_createDResources(void) -{ - dRessv07_t ress; - - /* init */ - ress.dctx = ZBUFFv07_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.dictBuffer = NULL; ress.dictBufferSize=0; - - /* Allocate Memory */ - ress.srcBufferSize = ZBUFFv07_recommendedDInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFFv07_recommendedDOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - return ress; -} - -static void FIOv07_freeDResources(dRessv07_t ress) -{ - size_t const errorCode = ZBUFFv07_freeDCtx(ress.dctx); - if (ZBUFFv07_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv07_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); -} - - -unsigned long long FIOv07_decompressFrame(dRessv07_t ress, - FILE* foutput, FILE* finput) -{ - U64 frameSize = 0; - size_t readSize = 4; - - MEM_writeLE32(ress.srcBuffer, ZSTDv07_MAGICNUMBER); - ZBUFFv07_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); - - while (1) { - /* Decode */ - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t toRead = ZBUFFv07_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFFv07_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv07_getErrorName(toRead)); - readSize -= inSize; - - /* Write block */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); } - frameSize += decodedSize; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); - - if (toRead == 0) break; - if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); - - /* Fill input buffer */ - if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); - readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); - } - - return frameSize; -} - - -/*===== General legacy dispatcher =====*/ - -unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, - const void* dictBuffer, size_t dictSize, - U32 magicNumberLE) -{ - switch(magicNumberLE) - { - case ZSTDv01_magicNumberLE : - return FIOv01_decompressFrame(foutput, finput); - case ZSTDv02_magicNumber : - return FIOv02_decompressFrame(foutput, finput); - case ZSTDv03_magicNumber : - return FIOv03_decompressFrame(foutput, finput); - case ZSTDv04_magicNumber : - { dRessv04_t r = FIOv04_createDResources(); - unsigned long long const s = FIOv04_decompressFrame(r, foutput, finput); - FIOv04_freeDResources(r); - return s; - } - case ZSTDv05_MAGICNUMBER : - { dRessv05_t r = FIOv05_createDResources(); - r.dictBuffer = dictBuffer; - r.dictBufferSize = dictSize; - { unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput); - FIOv05_freeDResources(r); - return s; - } } - case ZSTDv06_MAGICNUMBER : - { dRessv06_t r = FIOv06_createDResources(); - r.dictBuffer = dictBuffer; - r.dictBufferSize = dictSize; - { unsigned long long const s = FIOv06_decompressFrame(r, foutput, finput); - FIOv06_freeDResources(r); - return s; - } } - case ZSTDv07_MAGICNUMBER : - { dRessv07_t r = FIOv07_createDResources(); - r.dictBuffer = dictBuffer; - r.dictBufferSize = dictSize; - { unsigned long long const s = FIOv07_decompressFrame(r, foutput, finput); - FIOv07_freeDResources(r); - return s; - } } - default : - return ERROR(prefix_unknown); - } -} diff --git a/programs/legacy/fileio_legacy.h b/programs/legacy/fileio_legacy.h deleted file mode 100644 index 13b3c4a4..00000000 --- a/programs/legacy/fileio_legacy.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - fileio.h - file i/o handler - Copyright (C) Yann Collet 2013-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD source repository : https://github.com/Cyan4973/zstd - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* ************************************* -* Parameters -***************************************/ -void FIO_legacy_setNotificationLevel(unsigned level); - - -/* ************************************* -* Stream/File functions -***************************************/ -unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, - const void* dictBuffer, size_t dictSize, - U32 magicNumberLE); - - -#if defined (__cplusplus) -} -#endif diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 163e944c..26cd6c0c 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -45,10 +45,9 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRA IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy) - SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) ENDIF (ZSTD_LEGACY_SUPPORT) -ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY}) +ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c) TARGET_LINK_LIBRARIES(zstd libzstd_static) IF (UNIX) From 080940c62850b76087fdf16e532b3c31ae909d1b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 19:27:55 -0700 Subject: [PATCH 63/74] fixed Visual project, removing fileio_legacy --- projects/VS2005/zstd/zstd.vcproj | 12 ++++-------- projects/VS2008/zstd/zstd.vcproj | 12 ++++-------- projects/VS2010/zstd/zstd.vcxproj | 12 +++++------- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/projects/VS2005/zstd/zstd.vcproj b/projects/VS2005/zstd/zstd.vcproj index c9b7442b..ff45e393 100644 --- a/projects/VS2005/zstd/zstd.vcproj +++ b/projects/VS2005/zstd/zstd.vcproj @@ -43,7 +43,7 @@ - - diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj index 85a9f6bd..9d71f6ed 100644 --- a/projects/VS2008/zstd/zstd.vcproj +++ b/projects/VS2008/zstd/zstd.vcproj @@ -44,7 +44,7 @@ - - diff --git a/projects/VS2010/zstd/zstd.vcxproj b/projects/VS2010/zstd/zstd.vcxproj index 17b95567..5922d925 100644 --- a/projects/VS2010/zstd/zstd.vcxproj +++ b/projects/VS2010/zstd/zstd.vcxproj @@ -43,7 +43,6 @@ - @@ -68,7 +67,6 @@ - @@ -118,27 +116,27 @@ true - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ true - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ false - $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); @@ -219,4 +217,4 @@ - \ No newline at end of file + From 23b6e05d8eaf8725f0174e5937f93485fce637aa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 21:05:43 -0700 Subject: [PATCH 64/74] ZSTD_malloc() and ZSTD_free(), to simplify customMem --- lib/common/zstd_common.c | 17 ++++++- lib/common/zstd_internal.h | 3 ++ lib/compress/zstd_compress.c | 84 +++++++++++++++----------------- lib/decompress/zstd_decompress.c | 81 +++++++++++++++--------------- 4 files changed, 95 insertions(+), 90 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 0d9c1154..19e139a7 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -75,17 +75,30 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error +/*=************************************************************** +* Custom allocator +****************************************************************/ +/* default uses stdlib */ void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { void* address = malloc(size); (void)opaque; - /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } void ZSTD_defaultFreeFunction(void* opaque, void* address) { (void)opaque; - /* if (address) printf("free %p opaque=%p \n", address, opaque); */ free(address); } + +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +{ + return customMem.customAlloc(customMem.opaque, size); +} + +void ZSTD_free(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) + customMem.customFree(customMem.opaque, ptr); +} diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8b9b6290..259dc04f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -220,6 +220,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); +void ZSTD_free(void* ptr, ZSTD_customMem customMem); + /*====== common function ======*/ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1761a448..911dfeb0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -133,13 +133,10 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_CCtx* cctx; - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - cctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx)); + cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); if (!cctx) return NULL; memset(cctx, 0, sizeof(ZSTD_CCtx)); memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem)); @@ -149,8 +146,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support free on NULL */ - if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); - cctx->customMem.customFree(cctx->customMem.opaque, cctx); + ZSTD_free(cctx->workSpace, cctx->customMem); + ZSTD_free(cctx, cctx->customMem); return 0; /* reserved as a potential error code in the future */ } @@ -272,8 +269,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - zc->customMem.customFree(zc->customMem.opaque, zc->workSpace); - zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace); + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -2686,7 +2683,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS memset(&ctxBody, 0, sizeof(ctxBody)); memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ZSTD_free(ctxBody.workSpace, defaultCustomMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ return result; } @@ -2701,29 +2698,26 @@ struct ZSTD_CDict_s { ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) { - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; - if (!customMem.customAlloc || !customMem.customFree) /* can't have 1/2 custom alloc/free as NULL */ - return NULL; - - { ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict)); - void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); if (!dictContent || !cdict || !cctx) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, cdict); - customMem.customFree(customMem.opaque, cctx); + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); return NULL; } memcpy(dictContent, dict, dictSize); { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, cdict); - customMem.customFree(customMem.opaque, cctx); + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); return NULL; } } @@ -2744,12 +2738,13 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { - ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree; - void* const opaque = cdict->refContext->customMem.opaque; - ZSTD_freeCCtx(cdict->refContext); - cFree(opaque, cdict->dictContent); - cFree(opaque, cdict); - return 0; + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem cMem = cdict->refContext->customMem; + ZSTD_freeCCtx(cdict->refContext); + ZSTD_free(cdict->dictContent, cMem); + ZSTD_free(cdict, cMem); + return 0; + } } /*! ZSTD_compress_usingCDict() : @@ -2807,13 +2802,10 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) { ZSTD_CStream* zcs; - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream)); + zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); if (zcs==NULL) return NULL; memset(zcs, 0, sizeof(ZSTD_CStream)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); @@ -2825,11 +2817,13 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) size_t ZSTD_freeCStream(ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support free on NULL */ - ZSTD_freeCCtx(zcs->zc); - if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); - if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); - zcs->customMem.customFree(zcs->customMem.opaque, zcs); - return 0; + { ZSTD_customMem const cMem = zcs->customMem; + ZSTD_freeCCtx(zcs->zc); + ZSTD_free(zcs->inBuff, cMem); + ZSTD_free(zcs->outBuff, cMem); + ZSTD_free(zcs, cMem); + return 0; + } } @@ -2846,16 +2840,16 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zcs->inBuffSize < neededInBuffSize) { zcs->inBuffSize = neededInBuffSize; - zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); /* should not be necessary */ - zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize); + ZSTD_free(zcs->inBuff, zcs->customMem); /* should not be necessary */ + zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); if (zcs->inBuff == NULL) return ERROR(memory_allocation); } zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); } if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; - zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); /* should not be necessary */ - zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize); + ZSTD_free(zcs->outBuff, zcs->customMem); /* should not be necessary */ + zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); if (zcs->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e540e3fe..32917985 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -174,7 +174,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; - dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx)); + dctx = (ZSTD_DCtx*) ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); if (!dctx) return NULL; memcpy(&dctx->customMem, &customMem, sizeof(customMem)); ZSTD_decompressBegin(dctx); @@ -189,7 +189,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void) size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support free on NULL */ - dctx->customMem.customFree(dctx->customMem.opaque, dctx); + ZSTD_free(dctx, dctx->customMem); return 0; /* reserved as a potential error code in the future */ } @@ -1244,29 +1244,26 @@ struct ZSTD_DDict_s { ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) { - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); - void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); if (!dictContent || !ddict || !dctx) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); return NULL; } memcpy(dictContent, dict, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); if (ZSTD_isError(errorCode)) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); return NULL; } } @@ -1288,12 +1285,13 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) size_t ZSTD_freeDDict(ZSTD_DDict* ddict) { - ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree; - void* const opaque = ddict->refContext->customMem.opaque; - ZSTD_freeDCtx(ddict->refContext); - cFree(opaque, ddict->dict); - cFree(opaque, ddict); - return 0; + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->refContext->customMem; + ZSTD_freeDCtx(ddict->refContext); + ZSTD_free(ddict->dict, cMem); + ZSTD_free(ddict, cMem); + return 0; + } } /*! ZSTD_decompress_usingDDict() : @@ -1355,13 +1353,10 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) { ZSTD_DStream* zds; - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream)); + zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem); if (zds==NULL) return NULL; memset(zds, 0, sizeof(ZSTD_DStream)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); @@ -1375,18 +1370,18 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) size_t ZSTD_freeDStream(ZSTD_DStream* zds) { if (zds==NULL) return 0; /* support free on null */ - ZSTD_freeDCtx(zds->zd); - if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); - if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); - if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); + { ZSTD_customMem const cMem = zds->customMem; + ZSTD_freeDCtx(zds->zd); + ZSTD_free(zds->inBuff, cMem); + ZSTD_free(zds->outBuff, cMem); + ZSTD_free(zds->dictContent, cMem); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) - if (zds->legacyContext) { - ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); - zds->legacyContext = NULL; - } + if (zds->legacyContext) + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); #endif - zds->customMem.customFree(zds->customMem.opaque, zds); - return 0; + ZSTD_free(zds, cMem); + return 0; + } } @@ -1401,8 +1396,8 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ if (dictSize > zds->dictSize) { - if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); - zds->dictContent = zds->customMem.customAlloc(zds->customMem.opaque, dictSize); + ZSTD_free(zds->dictContent, zds->customMem); + zds->dictContent = ZSTD_malloc(dictSize, zds->customMem); if (zds->dictContent == NULL) return ERROR(memory_allocation); } memcpy(zds->dictContent, dict, dictSize); @@ -1515,15 +1510,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB size_t const neededOutSize = zds->fParams.windowSize + blockSize; zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { - zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + ZSTD_free(zds->inBuff, zds->customMem); zds->inBuffSize = blockSize; - zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize); + zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); if (zds->inBuff == NULL) return ERROR(memory_allocation); } if (zds->outBuffSize < neededOutSize) { - zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + ZSTD_free(zds->outBuff, zds->customMem); zds->outBuffSize = neededOutSize; - zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize); + zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); if (zds->outBuff == NULL) return ERROR(memory_allocation); } } zds->stage = zdss_read; From 6416b0d7052e48fe24206efcd26db06f9daa55de Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 29 Aug 2016 13:04:26 +0200 Subject: [PATCH 65/74] updated README.md --- lib/README.md | 2 +- programs/README.md | 4 ++-- projects/README.md | 4 ++-- projects/build/README.md | 3 +++ tests/README.md | 4 ++-- zlibWrapper/README.md | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/README.md b/lib/README.md index f6dbc418..efcbdc61 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,4 +1,4 @@ -zstd - library files +Zstandard library files ================================ The __lib__ directory contains several directories. diff --git a/programs/README.md b/programs/README.md index 9bd1e71b..203fd7b4 100644 --- a/programs/README.md +++ b/programs/README.md @@ -1,5 +1,5 @@ -zstd - Command Line Interface -================================ +Command Line Interface for Zstandard library +============================================ Command Line Interface (CLI) can be created using the `make` command without any additional parameters. There are however other Makefile targets that create different variations of CLI: diff --git a/projects/README.md b/projects/README.md index 6353623b..39742fb1 100644 --- a/projects/README.md +++ b/projects/README.md @@ -1,5 +1,5 @@ -projects for various integrated development environments (IDE) -================================ +Projects for various integrated development environments (IDE) +============================================================== #### Included projects diff --git a/projects/build/README.md b/projects/build/README.md index e1658493..3942e381 100644 --- a/projects/build/README.md +++ b/projects/build/README.md @@ -1,3 +1,6 @@ +Command line scripts for Visual Studio compilation without IDE +============================================================== + Here are a few command lines for reference : ### Build with Visual Studio 2013 for msvcr120.dll diff --git a/tests/README.md b/tests/README.md index 0d64c98d..79c067ab 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,5 @@ -programs and scripts for automated testing of zstd -================================ +Programs and scripts for automated testing of Zstandard +======================================================= This directory contains the following programs and scripts: - `datagen` : Synthetic and parametrable data generator, for tests diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 272f7b99..3a39f00a 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -1,4 +1,4 @@ -zstd wrapper for zlib +Zstandard wrapper for zlib ================================ The main objective of creating a zstd wrapper for [zlib](http://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib. From da9d3b7057eafb7e7e1ba4f3a1ac107dd170a213 Mon Sep 17 00:00:00 2001 From: David Lam Date: Mon, 29 Aug 2016 09:03:12 -0700 Subject: [PATCH 66/74] Cleanup some errors in typedef comments and remove duplicated HOWTO from zbuff_decompress.c --- lib/common/zbuff.h | 9 +++++++++ lib/compress/huf_compress.c | 2 +- lib/compress/zbuff_compress.c | 2 +- lib/compress/zstd_compress.c | 2 +- lib/decompress/zbuff_decompress.c | 26 +------------------------- lib/decompress/zstd_decompress.c | 4 ++-- 6 files changed, 15 insertions(+), 30 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 269dc227..3818af47 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -28,6 +28,15 @@ You can contact the author at : - zstd homepage : http://www.zstd.net/ */ + +/* *************************************************************** +* NOTES/WARNINGS +*****************************************************************/ +/* The streaming API defined here will soon be deprecated by the +* new one in 'zstd.h'; consider migrating towards newer streaming +* API. See 'lib/README.md'. +*****************************************************************/ + #ifndef ZSTD_BUFFERED_H_23987 #define ZSTD_BUFFERED_H_23987 diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 86a53c2e..ccf8e183 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -89,7 +89,7 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS struct HUF_CElt_s { U16 val; BYTE nbBits; -}; /* typedef'd to HUF_CElt within huf_static.h */ +}; /* typedef'd to HUF_CElt within "huf.h" */ typedef struct nodeElt_s { U32 count; diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 5d929185..acdb14aa 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -98,7 +98,7 @@ struct ZBUFF_CCtx_s { U32 checksum; U32 frameEnded; ZSTD_customMem customMem; -}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ +}; /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 911dfeb0..c60a1ee7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2694,7 +2694,7 @@ struct ZSTD_CDict_s { void* dictContent; size_t dictContentSize; ZSTD_CCtx* refContext; -}; /* typedef'd tp ZSTD_CDict within zstd.h */ +}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) { diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 908120fc..5211f541 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -40,30 +40,6 @@ #include "zbuff.h" -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFF_DCtx object is required to track streaming operations. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation, -* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be re-init multiple times. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; @@ -83,7 +59,7 @@ struct ZBUFF_DCtx_s { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; ZSTD_customMem customMem; -}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ +}; /* typedef'd to ZBUFF_DCtx within "zbuff.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 32917985..91bf0c81 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -145,7 +145,7 @@ struct ZSTD_DCtx_s size_t rleSize; BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } @@ -1240,7 +1240,7 @@ struct ZSTD_DDict_s { void* dict; size_t dictSize; ZSTD_DCtx* refContext; -}; /* typedef'd tp ZSTD_CDict within zstd.h */ +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) { From 14200a20f01fc25dfe997d7400513c38c18d984a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 30 Aug 2016 06:51:00 -0700 Subject: [PATCH 67/74] Fixed issue #304, reported by @borzunov --- lib/compress/huf_compress.c | 5 +++-- lib/compress/zstd_compress.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 86a53c2e..9f92d171 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -124,7 +124,8 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */ op[0] = (BYTE)size; return size+1; - } } + } + } /* raw values */ if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */ @@ -491,7 +492,7 @@ static size_t HUF_compress_internal ( /* Scan input and build symbol stats */ { size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); if (HUF_isError(largest)) return largest; - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* rle */ + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 911dfeb0..34c6cf3e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2540,8 +2540,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); /* known magic number : dict is parsed for entropy stats and content */ - { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8; - if (ZSTD_isError(eSize)) return eSize; + { size_t const eSize_8 = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); + size_t const eSize = eSize_8 + 8; + if (ZSTD_isError(eSize_8)) return eSize_8; return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); } } From 3b15f1f10f0596cb1d099a8316a5e506da36853f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 30 Aug 2016 09:56:25 -0700 Subject: [PATCH 68/74] minor refactor --- lib/dictBuilder/zdict.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index b2fb7c08..82676006 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -35,6 +35,7 @@ * Tuning parameters ****************************************/ #define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) +#define ZDICT_MIN_SAMPLES_SIZE 512 /*-************************************** @@ -78,11 +79,9 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define DICTLISTSIZE 10000 +#define DICTLISTSIZE_DEFAULT 10000 #define NOISELENGTH 32 -#define PRIME1 2654435761U -#define PRIME2 2246822519U #define MINRATIO 4 static const int g_compressionLevel_default = 5; @@ -560,10 +559,12 @@ _cleanup: static void ZDICT_fillNoise(void* buffer, size_t length) { - unsigned acc = PRIME1; + unsigned const prime1 = 2654435761U; + unsigned const prime2 = 2246822519U; + unsigned acc = prime1; size_t p=0;; for (p=0; p> 21); } } @@ -878,7 +879,6 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo } -#define DIB_MINSAMPLESSIZE 512 /*! ZDICT_trainFromBuffer_unsafe() : * Warning : `samplesBuffer` must be followed by noisy guard band. * @return : size of dictionary, or an error code which can be tested with ZDICT_isError() @@ -888,7 +888,7 @@ size_t ZDICT_trainFromBuffer_unsafe( const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t params) { - U32 const dictListSize = MAX(MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16)); + U32 const dictListSize = MAX(MAX(DICTLISTSIZE_DEFAULT, nbSamples), (U32)(maxDictSize/16)); dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); unsigned const selectivity = params.selectivityLevel == 0 ? g_selectivity_default : params.selectivityLevel; unsigned const minRep = (selectivity > 30) ? MINRATIO : nbSamples >> selectivity; @@ -899,7 +899,7 @@ size_t ZDICT_trainFromBuffer_unsafe( /* checks */ if (!dictList) return ERROR(memory_allocation); if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); } - if (samplesBuffSize < DIB_MINSAMPLESSIZE) { free(dictList); return 0; } /* not enough source to create dictionary */ + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return 0; } /* not enough source to create dictionary */ /* init */ ZDICT_initDictItem(dictList); @@ -990,7 +990,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit size_t result; void* newBuff; size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); - if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough content => no dictionary */ + if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */ newBuff = malloc(sBuffSize + NOISELENGTH); if (!newBuff) return ERROR(memory_allocation); From 4ded9e591cbed57c54fc8f7a50412af5980e23a7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 30 Aug 2016 10:04:33 -0700 Subject: [PATCH 69/74] added boilerplate --- CONTRIBUTING.md | 42 +++++++++++++++++++ LICENSE | 30 +++++++++++++ LICENSE-examples | 11 +++++ PATENTS | 33 +++++++++++++++ README.md | 19 +++++---- examples/dictionary_compression.c | 30 ++++--------- examples/dictionary_decompression.c | 29 ++++--------- examples/simple_compression.c | 29 ++++--------- examples/simple_decompression.c | 29 ++++--------- examples/streaming_compression.c | 30 ++++--------- examples/streaming_decompression.c | 31 ++++---------- lib/common/error_private.h | 39 ++++------------- lib/common/error_public.h | 39 ++++------------- lib/common/mem.h | 41 ++++-------------- lib/common/zbuff.h | 37 ++++------------ lib/common/zstd_common.c | 37 ++++------------ lib/common/zstd_internal.h | 38 ++++------------- lib/compress/zbuff_compress.c | 37 ++++------------ lib/compress/zstd_compress.c | 39 ++++------------- lib/compress/zstd_opt.h | 41 ++++-------------- lib/decompress/zbuff_decompress.c | 37 ++++------------ lib/decompress/zstd_decompress.c | 37 ++++------------ lib/dictBuilder/zdict.c | 39 ++++------------- lib/dictBuilder/zdict.h | 44 ++++++------------- lib/legacy/zstd_legacy.h | 39 ++++------------- lib/legacy/zstd_v01.c | 43 ++++--------------- lib/legacy/zstd_v01.h | 44 ++++++------------- lib/legacy/zstd_v02.c | 40 ++++-------------- lib/legacy/zstd_v02.h | 44 ++++++------------- lib/legacy/zstd_v03.c | 40 ++++-------------- lib/legacy/zstd_v03.h | 44 ++++++------------- lib/legacy/zstd_v04.c | 40 ++++-------------- lib/legacy/zstd_v04.h | 44 ++++++------------- lib/legacy/zstd_v05.c | 40 ++++-------------- lib/legacy/zstd_v05.h | 38 ++++------------- lib/legacy/zstd_v06.c | 40 ++++-------------- lib/legacy/zstd_v06.h | 38 ++++------------- lib/legacy/zstd_v07.c | 40 ++++-------------- lib/legacy/zstd_v07.h | 38 ++++------------- lib/zstd.h | 38 ++++------------- programs/bench.c | 31 ++++---------- programs/bench.h | 29 ++++--------- programs/datagen.c | 30 ++++--------- programs/datagen.h | 32 ++++---------- programs/dibio.c | 29 ++++--------- programs/dibio.h | 31 ++++---------- programs/fileio.c | 29 ++++--------- programs/fileio.h | 29 ++++--------- programs/util.h | 31 ++++---------- programs/zstdcli.c | 30 ++++--------- tests/datagencli.c | 32 ++++---------- tests/fullbench.c | 30 ++++--------- tests/fuzzer.c | 30 ++++--------- tests/namespaceTest.c | 33 ++++----------- tests/paramgrill.c | 31 ++++---------- tests/roundTripCrash.c | 30 ++++--------- tests/test-zstd-speed.py | 9 ++++ tests/test-zstd-versions.py | 11 ++++- tests/zbufftest.c | 30 ++++--------- tests/zstreamtest.c | 30 ++++--------- zlibWrapper/zstd_zlibwrapper.c | 65 ++++++++++------------------- zlibWrapper/zstd_zlibwrapper.h | 40 ++++-------------- 62 files changed, 616 insertions(+), 1524 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 LICENSE-examples create mode 100644 PATENTS diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..edf5b7f4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,42 @@ +# Contributing to Zstandard +We want to make contributing to this project as easy and transparent as +possible. + +## Our Development Process +New versions are being developed in the "dev" branch, +or in their own feature branch. +When they are deemed ready for a release, they are merged into "master". + +As a consequences, all contributions must stage first through "dev" +or their own feature branch. + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `dev`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints. +6. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is +clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Coding Style +* 4 spaces for indentation rather than tabs + +## License +By contributing to Zstandard, you agree that your contributions will be licensed +under the [LICENSE](LICENSE) file in the root directory of this source tree. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..a793a802 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE-examples b/LICENSE-examples new file mode 100644 index 00000000..1de78130 --- /dev/null +++ b/LICENSE-examples @@ -0,0 +1,11 @@ +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +The examples provided by Facebook are for non-commercial testing and evaluation +purposes only. Facebook reserves all rights not expressly granted. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +FACEBOOK BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/PATENTS b/PATENTS new file mode 100644 index 00000000..15b4a2ea --- /dev/null +++ b/PATENTS @@ -0,0 +1,33 @@ +Additional Grant of Patent Rights Version 2 + +"Software" means the Zstandard software distributed by Facebook, Inc. + +Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software +("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable +(subject to the termination provision below) license under any Necessary +Claims, to make, have made, use, sell, offer to sell, import, and otherwise +transfer the Software. For avoidance of doubt, no license is granted under +Facebook’s rights in any patent claims that are infringed by (i) modifications +to the Software made by you or any third party or (ii) the Software in +combination with any software or other technology. + +The license granted hereunder will terminate, automatically and without notice, +if you (or any of your subsidiaries, corporate affiliates or agents) initiate +directly or indirectly, or take a direct financial interest in, any Patent +Assertion: (i) against Facebook or any of its subsidiaries or corporate +affiliates, (ii) against any party if such Patent Assertion arises in whole or +in part from any software, technology, product or service of Facebook or any of +its subsidiaries or corporate affiliates, or (iii) against any party relating +to the Software. Notwithstanding the foregoing, if Facebook or any of its +subsidiaries or corporate affiliates files a lawsuit alleging patent +infringement against you in the first instance, and you respond by filing a +patent infringement counterclaim in that lawsuit against that party that is +unrelated to the Software, the license granted hereunder will not terminate +under section (i) of this paragraph due to such counterclaim. + +A "Necessary Claim" is a claim of a patent owned by Facebook that is +necessarily infringed by the Software standing alone. + +A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, +or contributory infringement or inducement to infringe any patent, including a +cross-claim or counterclaim. diff --git a/README.md b/README.md index bdea53c9..cf4a2ab3 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,6 @@ Hence, deploying one dictionary per type of data will provide the greater benefi #### Dictionary compression How To : -##### _Using the Command Line Utility_ : - 1) Create the dictionary `zstd --train FullPathToTrainingSet/* -o dictionaryName` @@ -77,14 +75,19 @@ Hence, deploying one dictionary per type of data will provide the greater benefi ### Status -Zstd compression format has reached "Final status". It means it is planned to become the official stable zstd format tagged `v1.0`. The reason it's not yet tagged `v1.0` is that it currently performs its "validation period", making sure the format holds all its promises and nothing was missed. -Zstd library also offers legacy decoder support. Any data compressed by any version >= `v0.1` is decodable now and in the future. -The library has been validated using strong [fuzzer tests](https://en.wikipedia.org/wiki/Fuzz_testing), including both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). It's able to withstand hazard situations, including invalid inputs. -As a consequence, Zstandard is considered safe for, and is currently used in, production environments. +Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amount of data in multiple formats and use cases. +Zstandard is considered safe for production environments. -### Branch Policy +### License -The "dev" branch is the one where all contributions will be merged before reaching "master". If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. Direct commit to "master" are not permitted. +Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS). + +### Contributing + +The "dev" branch is the one where all contributions will be merged before reaching "master". +If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. +Direct commit to "master" are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). ### Miscellaneous diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c index ef6bee06..08d639c0 100644 --- a/examples/dictionary_compression.c +++ b/examples/dictionary_compression.c @@ -1,27 +1,11 @@ -/* - Dictionary compression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // printf diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 54a583fa..b53d41ed 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -1,27 +1,12 @@ -/* - Dictionary decompression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // printf diff --git a/examples/simple_compression.c b/examples/simple_compression.c index adff81e8..c377b86c 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -1,27 +1,12 @@ -/* - Simple compression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // fprintf, perror diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index b907afa1..745dfd36 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -1,27 +1,12 @@ -/* - Simple decompression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // printf diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index e2568836..ec9c588a 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -1,27 +1,11 @@ -/* - Streaming compression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // fprintf, perror, feof diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c index a9efd7a2..d4dfacb2 100644 --- a/examples/streaming_decompression.c +++ b/examples/streaming_decompression.c @@ -1,33 +1,16 @@ -/* - Streaming compression - Educational program using zstd library - Copyright (C) Yann Collet 2016 +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #include // malloc, exit #include // fprintf, perror, feof #include // strerror #include // errno -#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being #include // presumes zstd library is installed diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 977fe3d4..d27e15af 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -1,35 +1,12 @@ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ /* Note : this module is expected to remain private, do not expose it */ #ifndef ERROR_H_MODULE diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 5a6d7107..d46abd2c 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -1,35 +1,12 @@ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ #ifndef ERROR_PUBLIC_H_MODULE #define ERROR_PUBLIC_H_MODULE diff --git a/lib/common/mem.h b/lib/common/mem.h index 6f31aac6..681dd35d 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -1,37 +1,12 @@ -/* ****************************************************************** - mem.h - low-level memory access routines - Copyright (C) 2013-2015, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ #ifndef MEM_H_MODULE #define MEM_H_MODULE diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 269dc227..41cb4c9b 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -1,33 +1,12 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ #ifndef ZSTD_BUFFERED_H_23987 #define ZSTD_BUFFERED_H_23987 diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 19e139a7..54bc91c8 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -1,33 +1,12 @@ -/* - Common functions of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ /*-************************************* diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 259dc04f..51e7170e 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -1,34 +1,12 @@ -/* - zstd_internal - common functions to include - Header File for include - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : https://www.zstd.net -*/ #ifndef ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 5d929185..157b3c8e 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -1,33 +1,12 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ /* ************************************* diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 34c6cf3e..7588de2a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1,35 +1,12 @@ -/* - ZSTD HC - High Compression Mode of Zstandard - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd source repository : https://www.zstd.net -*/ /*-******************************************************* diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index cb332d94..cb587290 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -1,35 +1,12 @@ -/* - ZSTD Optimal mode - Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd source repository : https://www.zstd.net -*/ /* Note : this file is intended to be included within zstd_compress.c */ @@ -500,7 +477,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); } - if (cur > last_pos || price <= opt[cur].price) + if (cur > last_pos || price <= opt[cur].price) SET_PRICE(cur, 1, 0, litlen, price); if (cur == last_pos) break; diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 908120fc..2a16b1ab 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -1,33 +1,12 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ /* ************************************* diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 32917985..e76ef976 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1,33 +1,12 @@ -/* - zstd - standard compression library - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ /* *************************************************************** * Tuning parameters diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 82676006..adfe55cf 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -1,35 +1,12 @@ -/* - dictBuilder - dictionary builder for zstd - Copyright (C) Yann Collet 2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd homepage : https://www.zstd.net -*/ /*-************************************** * Tuning parameters diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 68349da0..c84aedd1 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -1,35 +1,11 @@ -/* - dictBuilder header file - Copyright (C) Yann Collet 2016 - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd source repository : https://www.zstd.net -*/ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ #ifndef DICTBUILDER_H_001 #define DICTBUILDER_H_001 @@ -39,6 +15,10 @@ extern "C" { #endif +/*====== Dependencies ======*/ +#include /* size_t */ + + /*====== Export for Windows ======*/ /*! * ZSTD_DLL_EXPORT : diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 58985233..2a9f36aa 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -1,35 +1,12 @@ -/* - zstd_legacy - decoder for legacy format - Header File - Copyright (C) 2015-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ #ifndef ZSTD_LEGACY_H #define ZSTD_LEGACY_H diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 58951761..94847d5c 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1,37 +1,12 @@ -/* ****************************************************************** - ZSTD_v01 - Zstandard decoder, compatible with v0.1.x format - Copyright (C) 2013-2015, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ /****************************************** * Includes @@ -2170,5 +2145,3 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi } } - - diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h index 314cd349..0f2323db 100644 --- a/lib/legacy/zstd_v01.h +++ b/lib/legacy/zstd_v01.h @@ -1,36 +1,14 @@ -/* - zstd - standard compression library - Header File - Copyright (C) 2014-2015, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once +#ifndef ZSTD_V01_H_28739879432 +#define ZSTD_V01_H_28739879432 #if defined (__cplusplus) extern "C" { @@ -98,3 +76,5 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi #if defined (__cplusplus) } #endif + +#endif /* ZSTD_V01_H_28739879432 */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index 1d4abdf2..af136620 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2015, Yann Collet +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ #ifndef ERROR_H_MODULE #define ERROR_H_MODULE diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h index 1087b770..a371bd18 100644 --- a/lib/legacy/zstd_v02.h +++ b/lib/legacy/zstd_v02.h @@ -1,36 +1,14 @@ -/* - zstd_v02 - decoder for 0.2 format - Header File - Copyright (C) 2015, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once +#ifndef ZSTD_V02_H_4174539423 +#define ZSTD_V02_H_4174539423 #if defined (__cplusplus) extern "C" { @@ -97,3 +75,5 @@ size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSi #if defined (__cplusplus) } #endif + +#endif /* ZSTD_V02_H_4174539423 */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 17c43c78..637204de 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2015, Yann Collet +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ #ifndef ERROR_H_MODULE #define ERROR_H_MODULE diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h index 4546770b..8b89737b 100644 --- a/lib/legacy/zstd_v03.h +++ b/lib/legacy/zstd_v03.h @@ -1,36 +1,14 @@ -/* - zstd_v03 - decoder for 0.3 format - Header File - Copyright (C) 2015, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once +#ifndef ZSTD_V03_H_298734209782 +#define ZSTD_V03_H_298734209782 #if defined (__cplusplus) extern "C" { @@ -97,3 +75,5 @@ size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSi #if defined (__cplusplus) } #endif + +#endif /* ZSTD_V03_H_298734209782 */ diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index da9a1dcb..1239bdb3 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - zstd_v04.c - Decompression module for ZSTD v0.4 legacy format - Copyright (C) 2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ /*- Dependencies -*/ #include "zstd_v04.h" diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h index 9279ebf9..370553b1 100644 --- a/lib/legacy/zstd_v04.h +++ b/lib/legacy/zstd_v04.h @@ -1,36 +1,14 @@ -/* - zstd_v04 - decoder for 0.4 format - Header File - Copyright (C) 2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once +#ifndef ZSTD_V04_H_91868324769238 +#define ZSTD_V04_H_91868324769238 #if defined (__cplusplus) extern "C" { @@ -146,3 +124,5 @@ size_t ZBUFFv04_recommendedDOutSize(void); #if defined (__cplusplus) } #endif + +#endif /* ZSTD_V04_H_91868324769238 */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 2a08538e..96fdf35e 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - zstd_v05.c - Decompression module for ZSTD v0.5 legacy format - Copyright (C) 2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ /*- Dependencies -*/ #include "zstd_v05.h" diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h index 6dc17a64..da26d96c 100644 --- a/lib/legacy/zstd_v05.h +++ b/lib/legacy/zstd_v05.h @@ -1,34 +1,12 @@ -/* - zstd_v05 - decoder for 0.5 format - Header File - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ #ifndef ZSTDv05_H #define ZSTDv05_H diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index bc440c2c..6bd463b6 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - zstd_v06.c - Decompression module for ZSTD v0.6 legacy format - Copyright (C) 2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ /*- Dependencies -*/ #include "zstd_v06.h" diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 177f1483..bcc6efbc 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -1,34 +1,12 @@ -/* - zstd_v06 - decoder for 0.6 format - Header File - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ #ifndef ZSTDv06_H #define ZSTDv06_H diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 16c09af4..f948069e 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1,36 +1,12 @@ -/* ****************************************************************** - zstd_v07.c - Decompression module for ZSTD v0.7 legacy format - Copyright (C) 2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ /*- Dependencies -*/ #include /* size_t, ptrdiff_t */ diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h index 162566cd..d1fbc083 100644 --- a/lib/legacy/zstd_v07.h +++ b/lib/legacy/zstd_v07.h @@ -1,34 +1,12 @@ -/* - zstd_v07 - decoder for 0.7 format - Header File - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ #ifndef ZSTDv07_H_235446 #define ZSTDv07_H_235446 diff --git a/lib/zstd.h b/lib/zstd.h index 1f3dddb2..6302243a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1,34 +1,12 @@ -/* - zstd - standard compression library - Header File - Copyright (C) 2014-2016, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ #ifndef ZSTD_H_235446 #define ZSTD_H_235446 diff --git a/programs/bench.c b/programs/bench.c index dc36d170..3a290cc9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,27 +1,12 @@ -/* - bench.c - open-source compression benchmark module - Copyright (C) Yann Collet 2012-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net - - zstd source repository : https://github.com/Cyan4973/zstd -*/ /* ************************************* * Includes diff --git a/programs/bench.h b/programs/bench.h index 1a315640..7350fd43 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -1,26 +1,13 @@ -/* - bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : http://www.zstd.net/ -*/ #ifndef BENCH_H_121279284357 #define BENCH_H_121279284357 diff --git a/programs/datagen.c b/programs/datagen.c index d1fb858a..109b8e3d 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -1,27 +1,13 @@ -/* - datagen.c - compressible data generator test tool - Copyright (C) Yann Collet 2012-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ - - source repository : https://github.com/Cyan4973/zstd -*/ /* ************************************* * Compiler Options diff --git a/programs/datagen.h b/programs/datagen.h index de4a0740..55f9d828 100644 --- a/programs/datagen.h +++ b/programs/datagen.h @@ -1,27 +1,11 @@ -/* - datagen.h - compressible data generator header - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD source repository : https://github.com/Cyan4973/zstd - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ #include /* size_t */ diff --git a/programs/dibio.c b/programs/dibio.c index 19188d76..a793669c 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -1,26 +1,13 @@ -/* - dibio - I/O API for dictionary builder - Copyright (C) Yann Collet 2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ /*-************************************* * Includes diff --git a/programs/dibio.h b/programs/dibio.h index 8ad29202..6780d869 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -1,26 +1,11 @@ -/* - dibio.h - I/O API for dictionary builder - Copyright (C) Yann Collet 2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ /* This library is designed for a single-threaded console application. * It exit() and printf() into stderr when it encounters an error condition. */ diff --git a/programs/fileio.c b/programs/fileio.c index 55ba1425..b7b201e0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1,26 +1,13 @@ -/* - fileio.c - File i/o handler for zstd - Copyright (C) Yann Collet 2013-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ /* Note : this file is part of zstd command line, which is not library. The license of ZSTD library is BSD. diff --git a/programs/fileio.h b/programs/fileio.h index 06d977d6..66805f79 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -1,26 +1,13 @@ -/* - fileio.h - file i/o handler - Copyright (C) Yann Collet 2013-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : http://www.zstd.net/ -*/ #pragma once #if defined (__cplusplus) diff --git a/programs/util.h b/programs/util.h index 37ae4cb2..ee130f4b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -1,26 +1,12 @@ -/* ****************************************************************** - util.h - utility functions - Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet. +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : http://www.zstd.net/ -*/ #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -402,4 +388,3 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu #endif #endif /* UTIL_H_MODULE */ - diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 64d2c2a9..2d6d4521 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1,27 +1,13 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ -/* - zstdcli - Command Line Interface (cli) for zstd - Copyright (C) Yann Collet 2014-2016 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ /* Note : this is a user program, not part of libzstd. The license of libzstd is BSD. diff --git a/tests/datagencli.c b/tests/datagencli.c index c4fa7f73..772e3dc9 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -1,28 +1,12 @@ -/* - datagencli.c - compressible data command line generator - Copyright (C) Yann Collet 2012-2015 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD source repository : https://github.com/Cyan4973/zstd - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ /*-************************************ * Includes diff --git a/tests/fullbench.c b/tests/fullbench.c index f6852f6d..eaf1fc77 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -1,26 +1,12 @@ -/* - fullbench.c - Detailed bench program for zstd - Copyright (C) Yann Collet 2014-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ /*_************************************ * Includes diff --git a/tests/fuzzer.c b/tests/fuzzer.c index bed86f59..18948b91 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1,26 +1,12 @@ -/* - Fuzzer test tool for zstd - Copyright (C) Yann Collet 2014-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : http://www.zstd.net -*/ /*-************************************ * Compiler specific diff --git a/tests/namespaceTest.c b/tests/namespaceTest.c index 49d9b369..dd63186d 100644 --- a/tests/namespaceTest.c +++ b/tests/namespaceTest.c @@ -1,27 +1,12 @@ -/* ########################################################################## -# namespaceTest -# ensure xxhash namespace emulation is properly triggered -# Copyright (C) Yann Collet 2016 -# -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ -# ########################################################################*/ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + #include /* size_t */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 9348a40f..f253fa39 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1,26 +1,11 @@ -/* - paramgrill.c - parameter tester for zstd - Copyright (C) Yann Collet 2015-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ /*-************************************ diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c index d5b673f4..a296d416 100644 --- a/tests/roundTripCrash.c +++ b/tests/roundTripCrash.c @@ -1,26 +1,12 @@ -/* - roundTripCrash - Copyright (C) Yann Collet 2013-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ /* This program takes a file in input, performs a zstd round-trip test (compression - decompress) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index dba4f855..45cfe865 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -1,5 +1,14 @@ #! /usr/bin/env python +# +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# + import argparse import os import string diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index a0a179ce..5a24bc4d 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -1,7 +1,14 @@ #!/usr/bin/env python3 """Test zstd interoperability between versions""" -# Copyright Yann Collet, Przemyslaw Skibinski and Takayuki Matsuoka -# License GPLv2 + +# +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# import filecmp import glob diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 563a5c9a..52897993 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -1,26 +1,12 @@ -/* - Fuzzer test tool for ZBUFF - a buffered streaming API for ZSTD - Copyright (C) Yann Collet 2015-2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : https://www.zstd.net/ -*/ /*-************************************ * Compiler specific diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index dd00864d..f0219749 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1,26 +1,12 @@ -/* - Fuzzer test tool for zstd streaming API - Copyright (C) Yann Collet 2016 +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - ZSTD homepage : https://www.zstd.net/ -*/ /*-************************************ * Compiler specific diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 0218c6a2..9467950b 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -1,33 +1,12 @@ -/* - zstd_zlibwrapper.c - zstd wrapper for zlib - Copyright (C) 2016, Przemyslaw Skibinski. +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ #include /* vsprintf */ #include /* va_list, for z_gzprintf */ @@ -145,7 +124,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { ZWRAP_CCtx* zwc; - + if (!g_useZSTD) { LOG_WRAPPER("- deflateInit level=%d\n", level); return deflateInit_((strm), (level), version, stream_size); @@ -193,7 +172,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } } - + return Z_OK; } @@ -244,7 +223,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END; zwc->bytesLeft = bytesLeft; } - + if (flush == Z_SYNC_FLUSH) { size_t bytesLeft; size_t dstCapacity = strm->avail_out; @@ -260,7 +239,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } -ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) +ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) { if (!g_useZSTD) { LOG_WRAPPER("- deflateEnd\n"); @@ -307,7 +286,7 @@ typedef struct { ZBUFF_DCtx* zbd; char headerBuf[ZWRAP_HEADERSIZE]; int errorCount; - + /* zlib params */ int stream_size; char *version; @@ -396,7 +375,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, if (strm->state == NULL) return Z_MEM_ERROR; errorCode = ZBUFF_decompressInitDictionary(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } - + if (strm->total_in == ZSTD_frameHeaderSize_min) { size_t dstCapacity = 0; size_t srcSize = strm->total_in; @@ -414,7 +393,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, } -ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) +ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) { if (!strm->reserved) return inflate(strm, flush); @@ -455,7 +434,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; } if (strm->avail_in > 0) goto error; - + strm->next_in = strm2.next_in; strm->avail_in = strm2.avail_in; strm->next_out = strm2.next_out; @@ -504,7 +483,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (errorCode == 0) return Z_STREAM_END; return Z_OK; error: - ZWRAP_freeDCtx(zwd); + ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } @@ -517,7 +496,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) int ret = Z_OK; if (!strm->reserved) return inflateEnd(strm); - + LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; size_t const errorCode = ZWRAP_freeDCtx(zwd); @@ -711,7 +690,7 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, if (!g_useZSTD) return compress(dest, destLen, source, sourceLen); - { size_t dstCapacity = *destLen; + { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL); LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; @@ -727,8 +706,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, { if (!g_useZSTD) return compress2(dest, destLen, source, sourceLen, level); - - { size_t dstCapacity = *destLen; + + { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; @@ -752,7 +731,7 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, if (sourceLen < 4 || MEM_readLE32(source) != ZSTD_MAGICNUMBER) return uncompress(dest, destLen, source, sourceLen); - { size_t dstCapacity = *destLen; + { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index d14c3a92..24247b2c 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -1,33 +1,11 @@ -/* - zstd_zlibwrapper.h - zstd wrapper for zlib - Copyright (C) 2016, Przemyslaw Skibinski. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ #ifndef ZSTD_ZLIBWRAPPER_H #define ZSTD_ZLIBWRAPPER_H @@ -42,7 +20,7 @@ extern "C" { #if !defined(z_const) #if ZLIB_VERNUM >= 0x1260 - #define z_const const + #define z_const const #else #define z_const #endif From 599c69d917f48c94bd22e105f1d76dc7b4074a8a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 30 Aug 2016 13:33:20 -0700 Subject: [PATCH 70/74] minor Makefile updates --- Makefile | 31 ++++--------------------------- lib/Makefile | 47 ++++++++++++----------------------------------- programs/Makefile | 43 ++++++++++++++----------------------------- 3 files changed, 30 insertions(+), 91 deletions(-) diff --git a/Makefile b/Makefile index b7e0649c..0ca7714c 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,10 @@ # ################################################################ -# zstd - Makefile -# Copyright (C) Yann Collet 2014-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. # All rights reserved. # -# BSD license -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ################################################################ PRGDIR = programs diff --git a/lib/Makefile b/lib/Makefile index 579a906f..35522da6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,33 +1,10 @@ # ################################################################ -# ZSTD library - Makefile -# Copyright (C) Yann Collet 2015-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. # All rights reserved. # -# BSD license - -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# You can contact the author at : -# - ZSTD homepage : http://www.zstd.net +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ################################################################ # Version numbers @@ -125,14 +102,14 @@ install: libzstd libzstd.pc @echo zstd static and shared library installed uninstall: - @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) - @rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - @rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc - @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) - @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a - @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc + [ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) + @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/programs/Makefile b/programs/Makefile index 89859416..fc634b60 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -1,25 +1,10 @@ # ########################################################################## -# ZSTD programs - Makefile -# Copyright (C) Yann Collet 2015-2016 +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## # zstd : Command Line Utility, supporting gzip-like arguments # zstd32 : Same as zstd, but forced to compile in 32-bits mode @@ -111,7 +96,7 @@ zstd-pgo : clean zstd ./zstd -b $(PROFILE_WITH) ./zstd -b7i2 $(PROFILE_WITH) ./zstd -b5 $(PROFILE_WITH) - rm zstd + $(RM) zstd $(MAKE) zstd MOREFLAGS=-fprofile-use zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c @@ -131,8 +116,8 @@ zstd-small: clean clean: $(MAKE) -C ../lib clean - @rm -f ../lib/decompress/*.o - @rm -f core *.o tmp* result* *.gcda dictionary *.zst \ + @$(RM) ../lib/decompress/*.o + @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) @echo Cleaning completed @@ -154,11 +139,11 @@ install: zstd @echo zstd installation completed uninstall: - rm -f $(DESTDIR)$(BINDIR)/zstdcat - rm -f $(DESTDIR)$(BINDIR)/unzstd - [ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT) - rm -f $(DESTDIR)$(MANDIR)/zstdcat.1 - rm -f $(DESTDIR)$(MANDIR)/unzstd.1 - [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 + $(RM) $(DESTDIR)$(BINDIR)/zstdcat + $(RM) $(DESTDIR)$(BINDIR)/unzstd + [ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 + $(RM) $(DESTDIR)$(MANDIR)/unzstd.1 + [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && $(RM) $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled endif From 1c59c20903ecd2881c09dc8992fc4e56b955126e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 31 Aug 2016 07:15:44 -0700 Subject: [PATCH 71/74] removed redundant files --- lib/LICENSE | 26 ---- programs/COPYING | 339 ----------------------------------------------- 2 files changed, 365 deletions(-) delete mode 100644 lib/LICENSE delete mode 100644 programs/COPYING diff --git a/lib/LICENSE b/lib/LICENSE deleted file mode 100644 index 35495850..00000000 --- a/lib/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -ZSTD Library -Copyright (c) 2014-2015, Yann Collet -All rights reserved. - -BSD License - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/programs/COPYING b/programs/COPYING deleted file mode 100644 index d159169d..00000000 --- a/programs/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. From 5c33c32d5541fc4a83ff05dfddda9109878fcf0d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 31 Aug 2016 07:34:18 -0700 Subject: [PATCH 72/74] copy zlib license into file header --- zlibWrapper/examples/example.c | 25 +++++++++++++++++++++---- zlibWrapper/examples/example_original.c | 21 +++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index a2836cdb..bbb2cd5a 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -1,8 +1,25 @@ -/* the file contains minimal changes required to be compiled with zstd wrapper for zlib */ - /* example.c -- usage example of the zlib compression library - * Copyright (C) 1995-2006, 2011 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h + * the file contains minimal changes required to be compiled with zstd wrapper for zlib + */ + + /* + Copyright (c) 1995-2006, 2011 Jean-loup Gailly + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgement in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. */ /* @(#) $Id$ */ diff --git a/zlibWrapper/examples/example_original.c b/zlibWrapper/examples/example_original.c index 138a699b..649882bf 100644 --- a/zlibWrapper/examples/example_original.c +++ b/zlibWrapper/examples/example_original.c @@ -1,7 +1,24 @@ /* example.c -- usage example of the zlib compression library - * Copyright (C) 1995-2006, 2011 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h */ +/* + Copyright (c) 1995-2006, 2011 Jean-loup Gailly + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgement in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. +*/ /* @(#) $Id$ */ From 901e85fe265d59e43473cf43ab53436b24bef124 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 31 Aug 2016 07:51:25 -0700 Subject: [PATCH 73/74] version bump --- lib/zstd.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 6302243a..d768ded3 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -31,9 +31,9 @@ extern "C" { /*======= Version =======*/ -#define ZSTD_VERSION_MAJOR 0 -#define ZSTD_VERSION_MINOR 8 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 0 +#define ZSTD_VERSION_RELEASE 0 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 0588ee66cc003ab781c7ab3f57df04619c48e588 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 31 Aug 2016 09:05:22 -0700 Subject: [PATCH 74/74] updated NEWS --- NEWS | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index b8f8cca0..2e300726 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,11 @@ -v0.8.2 +v1.0.0 +Change Licensing, all project is now BSD, Copyright Facebook Small decompression speed improvement -API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}() -CLI Legacy support for formats v0.4+ +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew Enhanced documentation, by Przemyslaw Skibinski -Fixed : compression fails on specific huge files, bug introduced in v0.8.1, reported by Jesse McGrew v0.8.1 New streaming API