Merge pull request #2048 from nocnokneo/ctest-support

Add CTest support

.travis.yml

@@ -100,7 +100,7 @@ matrix:
         - make clean
         - make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
 
-    - name: cmake build and test check    # ~2mn
+    - name: cmake build and test check    # ~6mn
       script:
         - make cmakebuild
 

Makefile

@@ -362,7 +362,7 @@ cmakebuild:
 	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
 	$(MAKE) -C $(BUILDIR)/cmake/build install;
 	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
-	cd $(BUILDIR)/cmake/build; ctest -V
+	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
 
 c89build: clean
 	$(CC) -v

TESTING.md

@@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
 They consist of the following tests:
 - The following tests run with UBsan and Asan on x86_64 and x86, as well as with
   Msan on x86_64
-  - `tests/playTests.sh --test-long-data`
+  - `tests/playTests.sh --test-large-data`
   - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
 - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
 - Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)

appveyor.yml

@@ -249,9 +249,9 @@
       C:\cygwin64\bin\bash --login -c "
         set -e;
         cd build/cmake;
-        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON .;
+        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s .;
         make -j4;
-        ctest -V;
+        ctest -V -L Medium;
       "
     )
   - if [%HOST%]==[mingw] (

build/cmake/CMakeLists.txt

@@ -103,7 +103,14 @@ endif ()
 
 option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
 option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
-option(ZSTD_BUILD_TESTS "BUILD TESTS" OFF)
+
+# Respect the conventional CMake option for enabling tests if it was specified on the first configure
+if (BUILD_TESTING)
+    set(ZSTD_BUILD_TESTS_default ON)
+else()
+    set(ZSTD_BUILD_TESTS_default OFF)
+endif()
+option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
 if (MSVC)
     option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
 endif ()

build/cmake/tests/CMakeLists.txt

@@ -32,6 +32,16 @@
 
 project(tests)
 
+# name: Cache variable name. The value is expected to be a semicolon-separated
+# list of command line flags
+# default_value: Value to initialize the option with. Can be space separated.
+function(AddTestFlagsOption name default_value doc)
+    string(STRIP "${default_value}" default_value)
+    string(REGEX REPLACE " +" ";" default_value "${default_value}")
+    set(${name} ${default_value} CACHE STRING "${doc}")
+    mark_as_advanced(${name})
+endfunction()
+
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
 # Define programs directory, where sources and header files are located
@@ -43,18 +53,52 @@ include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c
 add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
 target_link_libraries(datagen libzstd_static)
 
+#
+# fullbench
+#
 add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
 target_link_libraries(fullbench libzstd_static)
+add_test(NAME fullbench COMMAND fullbench)
 
+#
+# fuzzer
+#
 add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
 target_link_libraries(fuzzer libzstd_static)
+AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
+    "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
+add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
+# Disable the timeout since the run time is too long for the default timeout of
+# 1500 seconds and varies considerably between low-end and high-end CPUs.
+set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
 
+#
+# zstreamtest
+#
 add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
 target_link_libraries(zstreamtest libzstd_static)
+AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
+    "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
+add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
 
-add_test(
-    NAME playTests
-    COMMAND sh -c "ZSTD_BIN='$<TARGET_FILE:zstd>' DATAGEN_BIN='$<TARGET_FILE:datagen>' '${TESTS_DIR}/playTests.sh'")
+#
+# playTests.sh
+#
+AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "--test-large-data"
+    "Semicolon-separated list of flags to pass to the playTests.sh test")
+add_test(NAME playTests COMMAND sh -c "${TESTS_DIR}/playTests.sh" ${ZSTD_PLAYTESTS_FLAGS})
+if (ZSTD_BUILD_PROGRAMS)
+    set_property(TEST playTests APPEND PROPERTY ENVIRONMENT
+        "ZSTD_BIN=$<TARGET_FILE:zstd>"
+        "DATAGEN_BIN=$<TARGET_FILE:datagen>"
+        )
+else()
+    message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled")
+    set_tests_properties(playTests PROPERTIES DISABLED YES)
+endif()
+
+# Label the "Medium" set of tests (see TESTING.md)
+set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
 
 if (UNIX)
     add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)

tests/fuzzer.c

@@ -3018,7 +3018,8 @@ static int FUZ_usage(const char* programName)
     DISPLAY( "      %s [args]\n", programName);
     DISPLAY( "\n");
     DISPLAY( "Arguments :\n");
-    DISPLAY( " -i#    : Nb of tests (default:%i) \n", nbTestsDefault);
+    DISPLAY( " -i#    : Number of tests (default:%i)\n", nbTestsDefault);
+    DISPLAY( " -T#    : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
     DISPLAY( " -t#    : Select starting test number (default:0)\n");
     DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);

tests/zstreamtest.c

@@ -2373,7 +2373,8 @@ static int FUZ_usage(const char* programName)
     DISPLAY( "      %s [args]\n", programName);
     DISPLAY( "\n");
     DISPLAY( "Arguments :\n");
-    DISPLAY( " -i#    : Nb of tests (default:%u) \n", nbTestsDefault);
+    DISPLAY( " -i#    : Number of tests (default:%u)\n", nbTestsDefault);
+    DISPLAY( " -T#    : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
     DISPLAY( " -t#    : Select starting test number (default:0)\n");
     DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);