From 4413ed27e450d7154ffbfa7c17b9313c2fd466fa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 16:27:24 -0800 Subject: [PATCH 01/10] circleci : make libzstd-nomt had its own test --- circle.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index b488c1a7..7e042d10 100644 --- a/circle.yml +++ b/circle.yml @@ -9,7 +9,7 @@ dependencies: test: override: - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean && make -C lib libzstd-nomt && make clean; fi && + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi : parallel: true @@ -58,6 +58,10 @@ test: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here : parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -C lib libzstd-nomt && make clean; fi + : + parallel: true post: - echo Circle CI tests finished From 519cf3b400b6f87578938ceff3762dc4d85ad07c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 16:31:40 -0800 Subject: [PATCH 02/10] circleci: add CFLAGS=-O0 environment variable for faster compilation --- circle.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/circle.yml b/circle.yml index 7e042d10..e7f17979 100644 --- a/circle.yml +++ b/circle.yml @@ -1,3 +1,7 @@ +machine: + environment: + CFLAGS: -O0 # for faster compilation, will preserve warnings + dependencies: override: - sudo dpkg --add-architecture i386 From 363355448b5032d062673cb5faf56a742571d190 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:02:55 -0800 Subject: [PATCH 03/10] circleci: removed one test node --- circle.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/circle.yml b/circle.yml index e7f17979..cc494905 100644 --- a/circle.yml +++ b/circle.yml @@ -59,11 +59,7 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -C lib libzstd-nomt && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi : parallel: true From 30eeb248a8e85dc60030003dee2677af8c6709c6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:22:04 -0800 Subject: [PATCH 04/10] added environment variable through circleci dashboard check if it works better --- circle.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/circle.yml b/circle.yml index cc494905..478775b7 100644 --- a/circle.yml +++ b/circle.yml @@ -1,7 +1,3 @@ -machine: - environment: - CFLAGS: -O0 # for faster compilation, will preserve warnings - dependencies: override: - sudo dpkg --add-architecture i386 From 74991c2764f32482815569a8efe4ee1e61bdfa71 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:28:24 -0800 Subject: [PATCH 05/10] circleci: removed gccX/clang tests does no longer need to install these large packages --- circle.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/circle.yml b/circle.yml index 478775b7..85061faa 100644 --- a/circle.yml +++ b/circle.yml @@ -3,7 +3,6 @@ dependencies: - sudo dpkg --add-architecture i386 - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 test: @@ -25,12 +24,7 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gcc6build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi : parallel: true - ? | @@ -43,11 +37,6 @@ test: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi : parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi - : - parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi From c6f5bf071cd3e246b3a4dae990489d3294e499eb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:35:19 -0800 Subject: [PATCH 06/10] fixed powerpc test misspelled CFLAGS --- Makefile | 2 +- circle.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9c265705..c8f0ed4d 100644 --- a/Makefile +++ b/Makefile @@ -161,7 +161,7 @@ aarch64build: clean CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd ppcbuild: clean - CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd + CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd ppc64build: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd diff --git a/circle.yml b/circle.yml index 85061faa..3b9facce 100644 --- a/circle.yml +++ b/circle.yml @@ -8,7 +8,7 @@ dependencies: test: override: - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean; fi && + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi : parallel: true From 71f012e5bfbebe91ca89d9fcc5700a26f0dadc87 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:42:46 -0800 Subject: [PATCH 07/10] zstdcli: fixed minor warning when bench module not enabled one variable defined but not used --- programs/zstdcli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 91af49e2..8b418620 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -773,8 +773,9 @@ int main(int argCount, const char* argv[]) BMK_setLdmHashEveryLog(g_ldmHashEveryLog); } BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); +#else + (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; #endif - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; goto _end; } From 6127592391bfda25154a5167954eda6b38e6efde Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 17:56:16 -0800 Subject: [PATCH 08/10] circleci : removed cmaketest strange error on pthread detection (seems a bug in cmake detection, since `make zstd` works perfectly with `-pthread` on same conf) --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 3b9facce..07f2990d 100644 --- a/circle.yml +++ b/circle.yml @@ -19,7 +19,7 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi : parallel: true - ? | @@ -44,7 +44,7 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # can add one more test here : parallel: true From 272c14233032eb02b79f8d2e69364a9e90ed7ae7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 18:11:34 -0800 Subject: [PATCH 09/10] circleci : balance some tests try to group them by "equivalent" duration since next job will not start as long as previous one is not completed on both VM --- circle.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/circle.yml b/circle.yml index 07f2990d..9b87ea56 100644 --- a/circle.yml +++ b/circle.yml @@ -19,11 +19,6 @@ test: parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi : parallel: true @@ -42,6 +37,11 @@ test: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi : parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi + : + parallel: true - ? | if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # can add one more test here From 6f1377301f7a33f39c9bbd9354730141f627733e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 1 Dec 2017 18:34:22 -0800 Subject: [PATCH 10/10] circleci: removed -m32 tests installing dependencies costs ~2mn, making it the longest test (test itself was < 15sec) --- circle.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/circle.yml b/circle.yml index 9b87ea56..ed50d59e 100644 --- a/circle.yml +++ b/circle.yml @@ -3,7 +3,6 @@ dependencies: - sudo dpkg --add-architecture i386 - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 test: override: @@ -23,13 +22,13 @@ test: : parallel: true - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make m32build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi : parallel: true - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi : parallel: true - ? | @@ -42,11 +41,6 @@ test: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi : parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # can add one more test here - : - parallel: true post: - echo Circle CI tests finished