Merge pull request #219 from inikep/dev

added gcc5test and gcc6test for Travis

.travis.yml

@@ -6,85 +6,82 @@ matrix:
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
     # Ubuntu 14.04 LTS Server Edition 64 bit
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
     # OS X Mavericks
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+      env: PLATFORM="OS X Mavericks" CMD="make test"
   exclude:
     - compiler: gcc
 
-before_install:
-  - set -e
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
-      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
-      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
-        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
-      fi
-    fi
-
 script:
-  - make $MAKE_PARAM
+  - sh -c "$CMD"

Makefile

@@ -51,7 +51,7 @@ all:
 
 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
-	mv $(PRGDIR)/zstd .
+	cp $(PRGDIR)/zstd .
 
 zlibwrapper:
 	$(MAKE) -C $(ZSTDDIR) all
@@ -87,6 +87,14 @@ travis-install:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
 clangtest: clean
 	clang -v
 	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -95,35 +103,13 @@ armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-# for Travis CI
-armtest-w-install: clean arminstall armtest
-
 ppctest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-ppcinstall: clean
-	# sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-# for Travis CI
-ppctest-w-install: clean ppcinstall ppctest
-
 ppc64test: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"
-
-ppc64install: clean
-	sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
-	update-binfmts --displ
-
-ppc64test-w-install: clean ppc64install ppc64test
 
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -183,3 +169,37 @@ bmix32test: clean
 bmi32test: clean
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
 endif
+
+
+#------------------------------------------------------------------------
+# for Travis CI
+#------------------------------------------------------------------------
+libc6install:
+	sudo apt-get install -y -qq libc6-dev-i386
+
+gppinstall:
+	sudo apt-get install -y -qq g++-multilib
+
+gcc5install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-5-multilib
+
+gcc6install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq 
+	sudo apt-get install -y -qq gcc-6-multilib
+
+arminstall: clean
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
+
+ppcinstall: clean
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
+
+ppc64install: clean   # compilation works but cannot be executed
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
+	update-binfmts --displ
+
+valgrindinstall:
+	echo sudo apt-get install -y -qq valgrind
+	sudo apt-get install -y -qq valgrind

programs/bench.c

@@ -42,10 +42,6 @@
 /* *************************************
 *  Constants
 ***************************************/
-#ifndef ZSTD_VERSION
-#  define ZSTD_VERSION ""
-#endif
-
 #define NBLOOPS               3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
@@ -363,7 +359,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     memset(&total, 0, sizeof(total));
 
     if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 

tests/test-zstd-speed.py

@@ -8,9 +8,9 @@ import time
 import traceback
 from subprocess import Popen, PIPE
 
-repo_url = 'https://github.com/Cyan4973/zstd.git'
+default_repo_url = 'https://github.com/Cyan4973/zstd.git'
 test_dir_name = 'speedTest'
-
+email_header = '[ZSTD_speedTest]'
 
 def log(text):
     print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
@@ -126,14 +126,23 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
         return text
 
 
+def send_simple_email(emails, email_topic, have_mutt, have_mail):
+    if have_mutt:
+        execute('mutt -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
+    elif have_mail:
+        execute('mail -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
+    else:
+        log("e-mail cannot be sent (mail and mutt not found)")
+
+
 def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
     with open(logFileName, "w") as myfile:
         myfile.writelines(text)
         myfile.close()
         if have_mutt:
-            execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
+            execute('mutt -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
         elif have_mail:
-            execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
+            execute('mail -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' < ' + logFileName)
         else:
             log("e-mail cannot be sent (mail and mutt not found)")
 
@@ -183,6 +192,7 @@ if __name__ == '__main__':
     parser.add_argument('testFileNames', help='file names list for speed benchmark')
     parser.add_argument('emails', help='list of e-mail addresses to send warnings')
     parser.add_argument('--message', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
     parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
     parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
     parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -197,7 +207,8 @@ if __name__ == '__main__':
         if os.path.isfile(fileName):
             testFilePaths.append(os.path.abspath(fileName))
         else:
-            raise RuntimeError("File not found: " + fileName)
+            log("ERROR: File not found: " + fileName)
+            exit(1)
 
     test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
     clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
@@ -206,19 +217,11 @@ if __name__ == '__main__':
     have_mutt = does_command_exist("mutt --help");
     have_mail = does_command_exist("mail -V");
     if not have_mutt and not have_mail:
-        log("WARNING: e-mail senders mail and mutt not found")
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
 
-    # clone ZSTD repo if needed
+    print "PARAMETERS:\nrepoURL=%s" % args.repoURL
-    if not os.path.isdir(test_path):
+    print "test_path=%s" % test_path
-        os.mkdir(test_path)
-    if not os.path.isdir(clone_path):
-        execute.cwd = test_path
-        execute('git clone ' + repo_url)
-    if not os.path.isdir(clone_path):
-        raise RuntimeError("ZSTD clone not found: " + clone_path)
-    execute.cwd = clone_path
-
-    print "PARAMETERS:\ntest_path=%s" % test_path
     print "clone_path=%s" % clone_path
     print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
     print "message=%s" % args.message
@@ -230,20 +233,34 @@ if __name__ == '__main__':
     print "dry_run=%s" % args.dry_run
     print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
 
+    # clone ZSTD repo if needed
+    if not os.path.isdir(test_path):
+        os.mkdir(test_path)
+    if not os.path.isdir(clone_path):
+        execute.cwd = test_path
+        execute('git clone ' + args.repoURL)
+    if not os.path.isdir(clone_path):
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
+    execute.cwd = clone_path
+
+    # check if speedTest.pid already exists
+    pid = str(os.getpid())
+    pidfile = "./speedTest.pid"
+    if os.path.isfile(pidfile):
+        log("ERROR: %s already exists, exiting" % pidfile)
+        exit(1)
+
+    send_simple_email(args.emails, "test-zstd-speed.py(%s) has been started" % pid, have_mutt, have_mail)
     while True:
-        pid = str(os.getpid())
+        file(pidfile, 'w').write(pid)
-        pidfile = "./speedTest.pid"
+        try:
-        if os.path.isfile(pidfile):
+            loadavg = os.getloadavg()[0]
-            log("%s already exists, exiting" % pidfile)
+            if (loadavg <= args.maxLoadAvg):
-        else:
+                check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
-            file(pidfile, 'w').write(pid)
+            else:
-            try:
+                log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
-                loadavg = os.getloadavg()[0]
+        finally:
-                if (loadavg <= args.maxLoadAvg):
+            os.unlink(pidfile)
-                    check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
-                else:
-                    log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
-            finally:
-                os.unlink(pidfile)
         log("sleep for %s seconds" % args.sleepTime)
         time.sleep(args.sleepTime)