From 993a9dfc5a0815b8e397b03cf5b6c5ba4191c5c5 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 10:07:46 +0200 Subject: [PATCH 01/14] playTests.sh updated to work better with /bin/sh --- programs/tests/playTests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/tests/playTests.sh b/programs/tests/playTests.sh index dbeb916d..5ff4c4e0 100755 --- a/programs/tests/playTests.sh +++ b/programs/tests/playTests.sh @@ -22,10 +22,12 @@ roundTripTest() { isWindows=false ECHO="echo" -if [[ "$OS" == "Windows"* ]]; then +case "$OS" in + Windows*) isWindows=true ECHO="echo -e" -fi + ;; +esac $ECHO "\nStarting playTests.sh isWindows=$isWindows" From e9dfdf6924e89f4af45309a1bb3bb70721501429 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 11:56:15 +0200 Subject: [PATCH 02/14] added zlibwrapper test for Travis --- .travis.yml | 3 ++- Makefile | 8 +++++++- zlibWrapper/Makefile | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f583a803..70f4bece 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,8 @@ env: - ZSTD_TRAVIS_CI_ENV=asan - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - + - ZSTD_TRAVIS_CI_ENV=zlibwrapper + compiler: gcc script: diff --git a/Makefile b/Makefile index 0894f75d..6be618bc 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ PRGDIR = programs ZSTDDIR = lib +ZWRAPDIR = zlibWrapper # Define nul output ifneq (,$(filter Windows%,$(OS))) @@ -40,7 +41,7 @@ else VOID = /dev/null endif -.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan +.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan default: zstdprogram @@ -51,12 +52,17 @@ all: zstdprogram: $(MAKE) -C $(PRGDIR) +zlibwrapper: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(ZWRAPDIR) all + test: $(MAKE) -C $(PRGDIR) $@ clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) @echo Cleaning completed diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index fd45ee6d..c74107d4 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -15,14 +15,14 @@ IMPLIB = -lz ../lib/libzstd.a endif ZLIBWRAPPER_PATH = . -EXAMPLE_PATH = examples/ +EXAMPLE_PATH = examples CC = gcc CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 LDFLAGS = $(LOC) RM = rm -f -all: example example_d +all: clean test testdll test: example ./example From d0c386247974c021f1119e6b90741676b9357a14 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 27 May 2016 12:33:19 +0200 Subject: [PATCH 03/14] #define z_const if not defined --- zlibWrapper/zstd_zlibwrapper.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index c9039ddb..c2c908dd 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -40,6 +40,9 @@ extern "C" { #define Z_PREFIX #include +#if !defined(z_const) +# define z_const const +#endif void useZSTD(int turn_on); int isUsingZSTD(); From 93fc13ebc8128c02bceb1c7548924d683bf8bd76 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:17:55 +0200 Subject: [PATCH 04/14] playTests.sh working with Linux, Windows, OS X --- appveyor.yml | 2 +- programs/Makefile | 2 +- programs/{tests => }/playTests.sh | 14 ++- programs/tests/playTests.bat | 157 ------------------------------ programs/tests/roundTripTest.bat | 8 -- programs/zstdcli.c | 2 +- 6 files changed, 13 insertions(+), 172 deletions(-) rename programs/{tests => }/playTests.sh (95%) mode change 100755 => 100644 delete mode 100644 programs/tests/playTests.bat delete mode 100644 programs/tests/roundTripTest.bat diff --git a/appveyor.yml b/appveyor.yml index 4721fa95..07c55964 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -93,7 +93,7 @@ test_script: CD programs && SET ZSTD=..\projects\zstd.exe && SET DATAGEN=..\projects\datagen.exe && - CALL tests\playTests.bat --test-large-data && + sh playTests.sh --test-large-data && CD .. && projects\fullbench.exe -i1 && projects\fullbench.exe -i1 -P0 && diff --git a/programs/Makefile b/programs/Makefile index c4e19229..f236459e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -204,7 +204,7 @@ endif #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) + ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) test: test-zstd test-fullbench test-fuzzer test-zbuff diff --git a/programs/tests/playTests.sh b/programs/playTests.sh old mode 100755 new mode 100644 similarity index 95% rename from programs/tests/playTests.sh rename to programs/playTests.sh index 5ff4c4e0..da1fee96 --- a/programs/tests/playTests.sh +++ b/programs/playTests.sh @@ -15,8 +15,8 @@ roundTripTest() { rm -f tmp1 tmp2 $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" - ./datagen $1 $p | md5sum > tmp1 - ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + ./datagen $1 $p | $MD5SUM > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2 diff -q tmp1 tmp2 } @@ -29,7 +29,13 @@ case "$OS" in ;; esac +MD5SUM="md5sum" +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + MD5SUM="md5 -r" +fi + $ECHO "\nStarting playTests.sh isWindows=$isWindows" +file $ZSTD [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" @@ -122,8 +128,8 @@ diff tmpSparse2M tmpSparseRegenerated $ECHO "\n**** dictionary tests **** " ./datagen > tmpDict -./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +./datagen -g1M | $MD5SUM > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 diff -q tmp1 tmp2 $ZSTD --train *.c *.h -o tmpDict $ZSTD xxhash.c -D tmpDict -of tmp diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat deleted file mode 100644 index 5f7ebd68..00000000 --- a/programs/tests/playTests.bat +++ /dev/null @@ -1,157 +0,0 @@ -@echo off -ECHO Start playTests.bat -ECHO ZSTD^: %ZSTD% -ECHO DATAGEN^: %DATAGEN% -if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 -if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 - -SET ROUNDTRIPTEST=tests\roundTripTest.bat - -echo. && echo **** simple tests **** -%DATAGEN% > tmp -%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst -%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) -echo test : too large compression level (must fail) -%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) -echo test : compress to stdout -%ZSTD% tmp -c > tmpCompressed -%ZSTD% tmp --stdout > tmpCompressed && REM long command format -echo test : null-length file roundtrip -echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) -echo test : decompress file with wrong suffix (must fail) -%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) -%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout -%ZSTD% --decompress tmpCompressed -c > tmpResult -%ZSTD% --decompress tmpCompressed --stdout > tmpResult -REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout -REM %ZSTD% -d - < tmp.zst > NUL -%ZSTD% -dc < tmp.zst > NUL -%ZSTD% -dc - < tmp.zst > NUL -%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) -%ZSTD% -q -f tmp -%ZSTD% -q --force tmp -%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) - -echo. && echo **** Pass-Through mode **** -echo "Hello world !" | %ZSTD% -df -echo "Hello world !" | %ZSTD% -dcf - - -echo. && echo **** frame concatenation **** - -echo hello > hello.tmp -echo world! > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -%ZSTD% -c hello.tmp > hello.zstd -%ZSTD% -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -%ZSTD% -dc helloworld.zstd > result.tmp -cat result.tmp -fc /b helloworld.tmp result.tmp -rm *.tmp *.zstd - -echo frame concatenation tests completed - - -REM echo. && echo **** flush write error test **** - -REM echo echo foo ^| %ZSTD% ^> v:\full -REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) -REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" -REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) - - -echo. && echo **** dictionary tests **** - -%DATAGEN% > tmpDict -%DATAGEN% -g1M | md5sum > tmp1 -%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 -fc tmp1 tmp2 -%ZSTD% --train *.c *.h -o tmpDict -%ZSTD% xxhash.c -D tmpDict -of tmp -%ZSTD% -d tmp -D tmpDict -of result -fc xxhash.c result - - -echo. && echo **** multiple files tests **** - -%DATAGEN% -s1 > tmp1 2> NUL -%DATAGEN% -s2 -g100K > tmp2 2> NUL -%DATAGEN% -s3 -g1M > tmp3 2> NUL -%ZSTD% -f tmp* -echo compress tmp* : -ls -ls tmp* -rm tmp1 tmp2 tmp3 -echo decompress tmp* : -%ZSTD% -df *.zst -ls -ls tmp* - -echo compress tmp* into stdout ^> tmpall : -%ZSTD% -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* -echo decompress tmpall* into stdout ^> tmpdec : -cp tmpall tmpall2 -%ZSTD% -dc tmpall* > tmpdec -ls -ls tmp* -echo compress multiple files including a missing one (notHere) : -%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) - - -echo. && echo **** integrity tests **** -echo test one file (tmp1.zst) -%ZSTD% -t tmp1.zst -%ZSTD% --test tmp1.zst -echo test multiple files (*.zst) -%ZSTD% -t *.zst -echo test good and bad files (*) -%ZSTD% -t * && (echo bad files not detected! && exit /b 1) - - -echo. && echo **** zstd round-trip tests **** - -CALL %ROUNDTRIPTEST% -CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 -CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 -CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 -CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 -CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain -CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 -CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt - -rm tmp* -echo Param = %1 -if NOT "%1"=="--test-large-data" ( - echo skipping large data tests - exit /b 0 -) - -CALL %ROUNDTRIPTEST% -g270000000 1 -CALL %ROUNDTRIPTEST% -g270000000 2 -CALL %ROUNDTRIPTEST% -g270000000 3 - -CALL %ROUNDTRIPTEST% -g140000000 -P60 4 -CALL %ROUNDTRIPTEST% -g140000000 -P60 5 -CALL %ROUNDTRIPTEST% -g140000000 -P60 6 - -CALL %ROUNDTRIPTEST% -g70000000 -P70 7 -CALL %ROUNDTRIPTEST% -g70000000 -P70 8 -CALL %ROUNDTRIPTEST% -g70000000 -P70 9 - -CALL %ROUNDTRIPTEST% -g35000000 -P75 10 -CALL %ROUNDTRIPTEST% -g35000000 -P75 11 -CALL %ROUNDTRIPTEST% -g35000000 -P75 12 - -CALL %ROUNDTRIPTEST% -g18000000 -P80 13 -CALL %ROUNDTRIPTEST% -g18000000 -P80 14 -CALL %ROUNDTRIPTEST% -g18000000 -P80 15 -CALL %ROUNDTRIPTEST% -g18000000 -P80 16 -CALL %ROUNDTRIPTEST% -g18000000 -P80 17 - -CALL %ROUNDTRIPTEST% -g50000000 -P94 18 -CALL %ROUNDTRIPTEST% -g50000000 -P94 19 - -CALL %ROUNDTRIPTEST% -g99000000 -P99 20 -CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 - -rm tmp* -exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat deleted file mode 100644 index 759891f0..00000000 --- a/programs/tests/roundTripTest.bat +++ /dev/null @@ -1,8 +0,0 @@ -@echo off -if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) -rm -f tmp1 tmp2 -echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -%DATAGEN% %1 %P% | md5sum > tmp1 -%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 -fc tmp1 tmp2 -EXIT /B %ERRORLEVEL% diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68dd98cb..18a81d66 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,7 +52,7 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE +#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else From 1981d5a7306d845df18708054560f8d1664170cd Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:18:56 +0200 Subject: [PATCH 05/14] Travis tests for PowerPC and OS X --- .travis.yml | 123 +++++++++++++++++++++++++++++++++++++++------------- Makefile | 27 +++++++----- 2 files changed, 108 insertions(+), 42 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70f4bece..f130978b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,96 @@ language: c - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq clang - - sudo apt-get install -qq g++-multilib - - sudo apt-get install -qq gcc-multilib - - sudo apt-get install -qq valgrind - -env: - - ZSTD_TRAVIS_CI_ENV=travis-install - - ZSTD_TRAVIS_CI_ENV=cmaketest - - ZSTD_TRAVIS_CI_ENV=clangtest - - ZSTD_TRAVIS_CI_ENV=gpptest - - ZSTD_TRAVIS_CI_ENV=gnu90test - - ZSTD_TRAVIS_CI_ENV=c99test - - ZSTD_TRAVIS_CI_ENV=gnu99test - - ZSTD_TRAVIS_CI_ENV=armtest-w-install - - ZSTD_TRAVIS_CI_ENV=test - - ZSTD_TRAVIS_CI_ENV="-C programs test32" - - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - - ZSTD_TRAVIS_CI_ENV="-C versionsTest" - - ZSTD_TRAVIS_CI_ENV=usan - - ZSTD_TRAVIS_CI_ENV=asan - - ZSTD_TRAVIS_CI_ENV=asan32 - - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - - ZSTD_TRAVIS_CI_ENV=zlibwrapper - compiler: gcc - -script: - - make $ZSTD_TRAVIS_CI_ENV - matrix: fast_finish: true + include: + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + # Standard Ubuntu 12.04 LTS Server Edition 64 bit + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + #- os: linux + # sudo: required + # env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper + # Ubuntu 14.04 LTS Server Edition 64 bit + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper + # OS X Mavericks + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=test + exclude: + - compiler: gcc + +before_install: + - set -e + - | + if [ "$TRAVIS_OS_NAME" == "linux" ]; then + CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l) + echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n" + if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then + sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind + fi + fi + +script: + - make $MAKE_PARAM diff --git a/Makefile b/Makefile index 6be618bc..3a585da3 100644 --- a/Makefile +++ b/Makefile @@ -120,11 +120,8 @@ armtest: clean $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -arminstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get install -q gcc-arm-linux-gnueabi +arminstall: clean + sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi # for Travis CI armtest-w-install: clean arminstall armtest @@ -134,16 +131,24 @@ ppctest: clean $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" # for Travis CI -ppcinstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get update -q - sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available) +ppcinstall: clean + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 # for Travis CI ppctest-w-install: clean ppcinstall ppctest +ppc64test: clean + $(MAKE) -C $(PRGDIR) datagen # use native, faster + $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + +ppc64install: clean + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu + update-binfmts --displ + +ppc64test-w-install: clean ppc64install ppc64test + usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" From 957d67803170585df6fc0b1aefa0bc3f91b40b7b Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 10:23:52 +0200 Subject: [PATCH 06/14] update playTests.sh perimissions --- programs/playTests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 programs/playTests.sh diff --git a/programs/playTests.sh b/programs/playTests.sh old mode 100644 new mode 100755 From 1dcb2ff23f6485c7ccfba49c6b164e79a61ae99e Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 30 May 2016 11:33:52 +0200 Subject: [PATCH 07/14] improved scripts compatibility --- appveyor.yml | 8 ++++---- programs/playTests.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 07c55964..724d0136 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ ) test_script: @@ -91,9 +91,9 @@ test_script: ) - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD programs && - SET ZSTD=..\projects\zstd.exe && - SET DATAGEN=..\projects\datagen.exe && - sh playTests.sh --test-large-data && + SET ZSTD=./zstd.exe && + SET DATAGEN=./datagen.exe && + sh -e playTests.sh --test-large-data && CD .. && projects\fullbench.exe -i1 && projects\fullbench.exe -i1 -P0 && diff --git a/programs/playTests.sh b/programs/playTests.sh index da1fee96..189d169f 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -30,15 +30,15 @@ case "$OS" in esac MD5SUM="md5sum" -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$TRAVIS_OS_NAME" = "osx" ]; then MD5SUM="md5 -r" fi -$ECHO "\nStarting playTests.sh isWindows=$isWindows" -file $ZSTD +$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" +file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp From d57b418214f47cd8ef1e9e44476555e3a676f114 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 11:46:16 +0200 Subject: [PATCH 08/14] updated appveyor.yml --- appveyor.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 724d0136..3d46ddc1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -92,11 +92,10 @@ test_script: - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD programs && SET ZSTD=./zstd.exe && - SET DATAGEN=./datagen.exe && sh -e playTests.sh --test-large-data && + fullbench.exe -i1 && + fullbench.exe -i1 -P0 && CD .. && - projects\fullbench.exe -i1 && - projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && From f772bf54a5649d6bcc741b2d78f11b7f37d9de65 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 12:43:46 +0200 Subject: [PATCH 09/14] support for skippable frames --- lib/common/zstd_internal.h | 2 +- lib/common/zstd_static.h | 4 +++- lib/decompress/zbuff_decompress.c | 24 +++++++++++++++++---- lib/decompress/zstd_decompress.c | 35 +++++++++++++++++++++++++++++-- programs/fileio.c | 2 +- programs/zbufftest.c | 11 +++++++--- 6 files changed, 66 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2eea5fee..b2fcd322 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -252,6 +252,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); - +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e0f50cf4..126d6cf5 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -51,7 +51,8 @@ extern "C" { /*-************************************* * Constants ***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U /*-************************************* @@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTD_frameHeaderSize_min = 5; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index d4ca550a..e785e9ec 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -218,12 +218,19 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + } + break; /* this was just a header */ + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -243,12 +250,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + break; + } + zbd->stage = ZBUFFds_read; /* this was just a header */ + break; + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b670c548..64be8fb5 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; struct ZSTD_DCtx_s { @@ -312,7 +313,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + fparamsPtr->frameContentSize = 0; + fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; + } + return ERROR(prefix_unknown); + } /* ensure there is enough `srcSize` to fully read/decode frame header */ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); @@ -995,6 +1004,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ @@ -1006,6 +1020,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case ZSTDds_getFrameHeaderSize : if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); @@ -1063,6 +1083,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->previousDstEnd = (char*)dst + rSize; return rSize; } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } default: return ERROR(GENERIC); /* impossible */ } diff --git a/programs/fileio.c b/programs/fileio.c index eba90d01..b333ded2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) continue; } #endif - if (magic != ZSTD_MAGICNUMBER) { + if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) { if (g_overwrite) /* -df : pass-through mode */ return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); else { diff --git a/programs/zbufftest.c b/programs/zbufftest.c index fce0ab27..8d32bbcc 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -146,7 +146,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); - size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); @@ -162,15 +163,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ - cSize = genSize; + cSize += genSize; genSize = compressedBufferSize - cSize; result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); if (result != 0) goto _output_error; /* error, or some data not flushed */ From 5bee32e34956e4eaa726d81fd185f62d385227b9 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 13:36:14 +0200 Subject: [PATCH 10/14] cmake project works with MSYS --- .gitignore | 2 +- programs/Makefile | 2 +- programs/dibio.h | 2 +- programs/zbufftest.c | 2 +- projects/cmake/lib/CMakeLists.txt | 23 ++++++++--------------- projects/cmake/programs/CMakeLists.txt | 8 ++------ 6 files changed, 14 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 2d3b6917..f0e2c9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,4 @@ _zstdbench/ zlib_wrapper/ # CMake -contrib/cmake/ +projects/cmake/ diff --git a/programs/Makefile b/programs/Makefile index f236459e..c57d8ec1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,7 +33,7 @@ DESTDIR?= PREFIX ?= /usr/local -CPPFLAGS= -I../lib/common +CPPFLAGS= -I../lib/common -I../lib/dictBuilder CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/programs/dibio.h b/programs/dibio.h index fd1f21df..0ccec413 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */ +#include "zdict_static.h" /* ZDICT_params_t */ /*-************************************* diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 8d32bbcc..6e5b7439 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -165,7 +165,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* generate skippable frame */ MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); - MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index dae706cb..c5557ff7 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -2,19 +2,19 @@ # zstd - Makefile # Copyright (C) Yann Collet 2014-2016 # All rights reserved. -# +# # BSD license # # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: -# +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. -# +# # * Redistributions in binary form must reproduce the above copyright notice, this # list of conditions and the following disclaimer in the documentation and/or # other materials provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -25,7 +25,7 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# +# # You can contact the author at : # - zstd homepage : http://www.zstd.net/ # ################################################################ @@ -105,11 +105,6 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v04.h) ENDIF (ZSTD_LEGACY_SUPPORT) -IF (MSVC) - SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib) - SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc) -ENDIF (MSVC) - # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -131,13 +126,11 @@ IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) # Define library base name -IF (UNIX) - SET(LIBRARY_BASE_NAME libzstd) -ELSEIF (MSVC) +IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) ELSE () - MESSAGE(FATAL_ERROR "Unsupported build type") -ENDIF (UNIX) + SET(LIBRARY_BASE_NAME libzstd) +ENDIF (MSVC) # Define static and shared library names SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME}) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 3b4f51d7..2670fc6a 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -38,14 +38,10 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) SET(ROOT_DIR ../../..) # Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) SET(PROGRAMS_DIR ${ROOT_DIR}/programs) -INCLUDE_DIRECTORIES(${PROGRAMS_DIR}) +INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder) -IF (WORKAROUND_OUTDATED_CODE_STYLE) - # Define library directory, where sources and header files are located - SET(LIBRARY_DIR ${ROOT_DIR}/lib) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common) -ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) From 8243b27f442e7d454338cc4a3eaa160b05849913 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 17:19:05 +0200 Subject: [PATCH 11/14] cmake project: removed WORKAROUND_OUTDATED_CODE_STYLE --- projects/cmake/CMakeLists.txt | 7 ------- projects/cmake/lib/CMakeLists.txt | 21 +++++++++------------ projects/cmake/programs/CMakeLists.txt | 7 +------ 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index 41883373..ae8f7be6 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT) ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0) ENDIF (ZSTD_LEGACY_SUPPORT) -# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate -IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE) -ELSE () - SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE) -ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index c5557ff7..40588874 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -95,16 +95,23 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v01.c ${LIBRARY_LEGACY_DIR}/zstd_v02.c ${LIBRARY_LEGACY_DIR}/zstd_v03.c - ${LIBRARY_LEGACY_DIR}/zstd_v04.c) + ${LIBRARY_LEGACY_DIR}/zstd_v04.c + ${LIBRARY_LEGACY_DIR}/zstd_v05.c) SET(Headers ${Headers} ${LIBRARY_LEGACY_DIR}/zstd_legacy.h ${LIBRARY_LEGACY_DIR}/zstd_v01.h ${LIBRARY_LEGACY_DIR}/zstd_v02.h ${LIBRARY_LEGACY_DIR}/zstd_v03.h - ${LIBRARY_LEGACY_DIR}/zstd_v04.h) + ${LIBRARY_LEGACY_DIR}/zstd_v04.h + ${LIBRARY_LEGACY_DIR}/zstd_v05.h) ENDIF (ZSTD_LEGACY_SUPPORT) +IF (MSVC) + SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib) + SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc) +ENDIF (MSVC) + # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -115,16 +122,6 @@ IF (MSVC) SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") ENDIF (MSVC) -# Define include directories -IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common) - IF (ZSTD_LEGACY_SUPPORT) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR}) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR}) - ENDIF (ZSTD_LEGACY_SUPPORT) -ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - # Define library base name IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index 2670fc6a..1cf1b69e 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -45,12 +45,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBui IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) - INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR}) - - IF (WORKAROUND_OUTDATED_CODE_STYLE) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy) - ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) - + INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy) SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) ENDIF (ZSTD_LEGACY_SUPPORT) From 019b83330646a50eec928166633cdece852e6d65 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 17:34:37 +0200 Subject: [PATCH 12/14] VS project: dictBuilder added to include directories --- projects/VS2008/zstd/zstd.vcproj | 8 ++++---- projects/VS2008/zstdlib/zstdlib.vcproj | 8 ++++---- projects/VS2010/zstd/zstd.vcxproj | 10 +++++----- projects/VS2010/zstdlib/zstdlib.vcxproj | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj index 9e462650..f84fbc93 100644 --- a/projects/VS2008/zstd/zstd.vcproj +++ b/projects/VS2008/zstd/zstd.vcproj @@ -44,7 +44,7 @@ + @@ -119,27 +119,27 @@ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index e8625a50..9e497811 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -1,4 +1,4 @@ - + @@ -97,28 +97,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false true zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false From 43aa9fe8b3f6b48e1ab5ad4da7d8b8a28b47d181 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 19:36:51 +0200 Subject: [PATCH 13/14] fixed skippable frame --- lib/decompress/zbuff_decompress.c | 20 +++------------- lib/decompress/zstd_decompress.c | 2 +- programs/zbufftest.c | 38 +++++++++++++++++++------------ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index e785e9ec..72456fd5 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -224,13 +224,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) { - if (isSkipFrame) { - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = 0; - } - break; /* this was just a header */ - } + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -256,15 +250,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { - if (isSkipFrame) { - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = 0; - break; - } - zbd->stage = ZBUFFds_read; /* this was just a header */ - break; - } + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ @@ -292,7 +278,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, *srcSizePtr = ip-istart; *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ +// if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbd->inPos; /* already loaded*/ return nextSrcSizeHint; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 64be8fb5..75e91e81 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1090,7 +1090,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; } case ZSTDds_skipFrame: - { dctx->expected = ZSTD_frameHeaderSize_min; + { dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; return 0; } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 6e5b7439..5b172d23 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -151,7 +151,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - size_t result, cSize, readSize, genSize; + size_t result, cSize, readSize, readSkipSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); @@ -185,12 +185,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - readSize = cSize; + readSkipSize = cSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); + if (genSize != 0) goto _output_error; /* skippable frame */ + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + readSize = cSize - readSkipSize; + genSize = CNBufferSize; + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer+readSkipSize, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ @@ -204,17 +209,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); - ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - { size_t r = 1, pIn=0, pOut=0; - while (r) { - size_t inS = 1; - size_t outS = 1; - r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); - pIn += inS; - pOut += outS; - } - readSize = pIn; - genSize = pOut; + { size_t r, pIn=0, pOut=0; + do + { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + r = 1; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } while (genSize==0); } if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */ From cf6edcd0a13fd7f2dc85384e3883886535960d23 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 31 May 2016 20:21:43 +0200 Subject: [PATCH 14/14] zbufftest.c: fixed g++ warning --- programs/zbufftest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 5b172d23..552d0928 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -192,7 +192,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); readSize = cSize - readSkipSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer+readSkipSize, &readSize); + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */