diff --git a/.travis.yml b/.travis.yml index 9c1e10e1..a52d57af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,39 +1,39 @@ +# Medium Tests: Run on all commits/PRs to dev branch + language: c sudo: required dist: trusty matrix: - fast_finish: true include: # Ubuntu 14.04 - - env: Cmd="make libc6install && make -C tests test32" - - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' + - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - - env: Cmd='CC=gcc-6 make gcc6install uasan-test' - - env: Cmd='CC=gcc-6 make gcc6install uasan-test32' - - env: Cmd="make arminstall armtest && make clean && make aarch64test" - - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test' - - env: Cmd='make gpp6install zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest && make clean && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - install: - - export CXX="g++-6" CC="gcc-6" + - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' + - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest' + - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' + - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - # OS X Mavericks - - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install" - os: osx + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' + + - env: Cmd='make arminstall && make armfuzz' + - env: Cmd='make arminstall && make aarch64fuzz' + - env: Cmd='make ppcinstall && make ppcfuzz' + - env: Cmd='make ppcinstall && make ppc64fuzz' + +git: + depth: 1 + +branches: + only: + - dev + - master script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # cron & master => full tests, as this is the final step towards a Release - # pull requests => normal tests (job numbers 1-3) - # other feature branches => short tests (job numbers 1-2) - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then - FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; - else - if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then - sh -c "$Cmd" || travis_terminate 1; - else - if [ $JOB_NUMBER -lt 3 ]; then - sh -c "$Cmd" || travis_terminate 1; - fi - fi - fi + - export FUZZERTEST=-T2mn; + export ZSTREAM_TESTTIME=-T2mn; + export DECODECORPUS_TESTTIME=-T1mn; + sh -c "$Cmd" || travis_terminate 1; diff --git a/Makefile b/Makefile index e10d2926..49f29d78 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,18 @@ ppcbuild: clean ppc64build: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch +armfuzz: clean + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +aarch64fuzz: clean + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +ppcfuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +ppc64fuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest + gpptest: clean CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" @@ -188,24 +200,36 @@ arm-ppc-compilation: $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" +# run UBsan with -fsanitize-recover=signed-integer-overflow +# due to a bug in UBsan when doing pointer subtraction +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303 + usan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined" asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" +asan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $* + msan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason +msan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $* + asan32: clean $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" uasan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" uasan-%: clean - LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $* +tsan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* apt-install: sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) @@ -217,7 +241,7 @@ ppcinstall: APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install arminstall: - APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install valgrindinstall: APT_PACKAGES="valgrind" $(MAKE) apt-install @@ -231,6 +255,9 @@ gcc6install: apt-add-repo gpp6install: apt-add-repo APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install +clang38install: + APT_PACKAGES="clang-3.8" $(MAKE) apt-install + endif diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000..1fa5fe8c --- /dev/null +++ b/TESTING.md @@ -0,0 +1,44 @@ +Testing +======= + +Zstandard CI testing is split up into three sections: +short, medium, and long tests. + +Short Tests +----------- +Short tests run on CircleCI for new commits on every branch and pull request. +They consist of the following tests: +- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64) +- Compilation on various versions of gcc, clang, and g++ +- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests) +- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64 + +Medium Tests +------------ +Medium tests run on every commit and pull request to `dev` branch, on TravisCI. +They consist of the following tests: +- The following tests run with UBsan and Asan on x86_64 and x86, as well as with + Msan on x86_64 + - `tests/playTests.sh --test-long-data` + - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c` +- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode) +- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind) +- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64 + +Long Tests +---------- +Long tests run on all commits to `master` branch, +and once a day on the current version of `dev` branch, +on TravisCI. +They consist of the following tests: +- Entire test suite (including fuzzers and some other specialized tests) on: + - x86_64 and x86 with UBsan and Asan + - x86_64 with Msan + - ARM, AArch64, PowerPC, and PowerPC64 +- Streaming mode fuzzer with Tsan (for the `zstdmt` testing) +- ZlibWrapper tests, including under valgrind +- Versions test (ensuring `zstd` can decode files from all previous versions) +- `pzstd` with asan and tsan, as well as in 32-bits mode +- Testing `zstd` with legacy mode off +- Testing `zbuff` (old streaming API) +- Entire test suite and make install on OS X diff --git a/appveyor.yml b/appveyor.yml index 51ff488a..9507fec6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -146,6 +146,11 @@ test_script: fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) +branches: + only: + - dev + - master + artifacts: - path: bin\zstd.exe - path: bin\zstd32.exe diff --git a/tests/Makefile b/tests/Makefile index 8b19aa3d..59256f84 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -60,10 +60,10 @@ endif MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD) VOID = /dev/null -ZSTREAM_TESTTIME = -T2mn +ZSTREAM_TESTTIME ?= -T2mn FUZZERTEST ?= -T5mn ZSTDRTTEST = --test-large-data -DECODECORPUS_TESTTIME = -T30 +DECODECORPUS_TESTTIME ?= -T30 .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest @@ -75,8 +75,6 @@ all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32 dll: fuzzer-dll zstreamtest-dll zbufftest-dll - - zstd: $(MAKE) -C $(PRGDIR) $@ @@ -257,6 +255,8 @@ zstd-playTests: datagen shortest: ZSTDRTTEST= shortest: test-zstd +fuzztest: test-fuzzer test-zstream test-decodecorpus + test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool