open-source-parsers
/
jsoncpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ENH: Refactor and enhance the CI testing infrastructure 

1) Improve travis build script for use outside travis.
   Allow the script used for CI builds to also be used
   locally in a similar manner to the CI use of the scrips

2) Add ctest compatible testing and CDASH support
   Report testing and building results to
   https://my.cdash.org/index.php?project=jsoncpp

   NOTE: The new ctest infrastructure is not yet robust on winodws
         Do no yet enable the new features for running test with ctest
         on windows platform.  The previous behaviors are maintainted,
         but enhance test reporting from windows is not yet supported.

3) Add a cmake coverage testing option
   Ensure that cmake builds on linux are tested.
   Ensure that code coverage is reported.

4) Move conditional environment checking into the matrix
   Avoid multiple places where conditional logic is used to
   change compiler behavior.  As more test environments are
   created fromt the travis.yml matrix, all settings should be
   obvious from that one location.

5) Tests with known regressions from the jsonchecker are suppressed
    Tests that are known to pass with jsoncpp more lenient
    syntax enforcement are exluded from tests in test/runjsontests.py
master
Hans Johnson 2019-01-12 12:32:15 -06:00 committed by Hans Johnson
parent 10a1a38b37
commit a3c8e86c0b
12 changed files with 341 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
.travis.yml
View File

@ -26,19 +26,52 @@ matrix:
allow_failures:
- os: osx
include:
- os: osx
- name: Mac clang meson static release testing
os: osx
osx_image: xcode9.4
compiler: clang
env: LIB_TYPE=static BUILD_TYPE=release
- os: linux
env:
CXX="clang++-3.5"
CC="clang-3.5"
LIB_TYPE=static
BUILD_TYPE=release
script: ./.travis_scripts/meson_builder.sh
- name: trusty clang meson static release testing
os: linux
dist: trusty
compiler: clang
env: LIB_TYPE=static BUILD_TYPE=release
env:
CXX="clang++-3.5"
CC="clang-3.5"
LIB_TYPE=static
BUILD_TYPE=release
# before_install and install steps only needed for linux meson builds
before_install:
- source ./.travis_scripts/travis.before_install.${TRAVIS_OS_NAME}.sh
install:
- source ./.travis_scripts/travis.install.${TRAVIS_OS_NAME}.sh
script: ./.travis_scripts/meson_builder.sh
- name: xenial gcc cmake coverage
os: linux
dist: xenial
compiler: gcc
env:
CXX=g++
CC=gcc
DO_Coverage=ON
BUILD_TOOL="Unix Makefiles"
BUILD_TYPE=Debug
LIB_TYPE=shared
DESTDIR=/tmp/cmake_json_cpp
script: ./.travis_scripts/cmake_builder.sh
# Valgrind has too many false positives from the python wrapping. Need a good suppression file
# - name: xenial gcc cmake coverage
# os: linux
# dist: xenial
# compiler: gcc
# env: DO_MemCheck=ON CXX=/usr/bin/g++ BUILD_TOOL="Unix Makefiles" BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp
# script: ./.travis_scripts/cmake_builder.sh
notifications:
email: false
before_install:
- source ./.travis_scripts/travis.before_install.${TRAVIS_OS_NAME}.sh
install:
- source ./.travis_scripts/travis.install.${TRAVIS_OS_NAME}.sh
script: ./.travis_scripts/travis.sh

130
.travis_scripts/cmake_builder.sh Executable file
View File

@ -0,0 +1,130 @@
#!/usr/bin/env sh
# This script can be used on the command line directly to configure several
# different build environments.
# This is called by `.travis.yml` via Travis CI.
# Travis supplies $TRAVIS_OS_NAME.
# http://docs.travis-ci.com/user/multi-os/
# Our .travis.yml also defines:
# - BUILD_TYPE=Release/Debug
# - LIB_TYPE=static/shared
#
# Optional environmental variables
# - DESTDIR <- used for setting the install prefix
# - BUILD_TOOL=["Unix Makefile"|"Ninja"]
# - BUILDNAME <-- how to identify this build on the dashboard
# - DO_MemCheck <- if set, try to use valgrind
# - DO_Coverage <- if set, try to do dashboard coverage testing
#
env_set=1
if ${BUILD_TYPE+false}; then
echo "BUILD_TYPE not set in environment."
env_set=0
fi
if ${LIB_TYPE+false}; then
echo "LIB_TYPE not set in environment."
env_set=0
fi
if ${CXX+false}; then
echo "CXX not set in environment."
env_set=0
fi
if [ ${env_set} -eq 0 ]; then
echo "USAGE: CXX=$(which clang++) BUILD_TYPE=[Release|Debug] LIB_TYPE=[static|shared] $0"
echo ""
echo "Examples:"
echo " CXX=$(which clang++) BUILD_TYPE=Release LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=Release LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=Debug LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=Release LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=Release LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=Debug LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0"
exit -1
fi
if ${DESTDIR+false}; then
DESTDIR="/usr/local"
fi
# -e: fail on error
# -v: show commands
# -x: show expanded commands
set -vex
env | sort
which cmake
cmake --version
echo ${CXX}
${CXX} --version
_COMPILER_NAME=`basename ${CXX}`
if [ "${BUILD_TYPE}" == "shared" ]; then
_CMAKE_BUILD_SHARED_LIBS=ON
else
_CMAKE_BUILD_SHARED_LIBS=OFF
fi
CTEST_TESTING_OPTION="-D ExperimentalTest"
# - DO_MemCheck <- if set, try to use valgrind
if ! ${DO_MemCheck+false}; then
valgrind --version
CTEST_TESTING_OPTION="-D ExperimentalMemCheck"
else
# - DO_Coverage <- if set, try to do dashboard coverage testing
if ! ${DO_Coverage+false}; then
export CXXFLAGS="-fprofile-arcs -ftest-coverage"
export LDFLAGS="-fprofile-arcs -ftest-coverage"
CTEST_TESTING_OPTION="-D ExperimentalTest -D ExperimentalCoverage"
#gcov --version
fi
fi
# Ninja = Generates build.ninja files.
if ${BUILD_TOOL+false}; then
BUILD_TOOL="Ninja"
export _BUILD_EXE=ninja
which ninja
ninja --version
else
# Unix Makefiles = Generates standard UNIX makefiles.
export _BUILD_EXE=make
fi
_BUILD_DIR_NAME="build-cmake_${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}_${_BUILD_EXE}"
mkdir -p ${_BUILD_DIR_NAME}
cd "${_BUILD_DIR_NAME}"
if ${BUILDNAME+false}; then
_HOSTNAME=`hostname -s`
BUILDNAME="${_HOSTNAME}_${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}_${_BUILD_EXE}"
fi
cmake \
-G "${BUILD_TOOL}" \
-DBUILDNAME:STRING="${BUILDNAME}" \
-DCMAKE_CXX_COMPILER:PATH=${CXX} \
-DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE} \
-DBUILD_SHARED_LIBS:BOOL=${_CMAKE_BUILD_SHARED_LIBS} \
-DCMAKE_INSTALL_PREFIX:PATH=${DESTDIR} \
../
ctest -C ${BUILD_TYPE} -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild ${CTEST_TESTING_OPTION} -D ExperimentalSubmit
# Final step is to verify that installation succeeds
cmake --build . --config ${BUILD_TYPE} --target install
if [ "${DESTDIR}" != "/usr/local" ]; then
${_BUILD_EXE} install
fi
cd -
if ${CLEANUP+false}; then
echo "Skipping cleanup: build directory will persist."
else
rm -r "${_BUILD_DIR_NAME}"
fi

81
.travis_scripts/meson_builder.sh Executable file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env sh
# This script can be used on the command line directly to configure several
# different build environments.
# This is called by `.travis.yml` via Travis CI.
# Travis supplies $TRAVIS_OS_NAME.
# http://docs.travis-ci.com/user/multi-os/
# Our .travis.yml also defines:
# - BUILD_TYPE=release/debug
# - LIB_TYPE=static/shared
env_set=1
if ${BUILD_TYPE+false}; then
echo "BUILD_TYPE not set in environment."
env_set=0
fi
if ${LIB_TYPE+false}; then
echo "LIB_TYPE not set in environment."
env_set=0
fi
if ${CXX+false}; then
echo "CXX not set in environment."
env_set=0
fi
if [ ${env_set} -eq 0 ]; then
echo "USAGE: CXX=$(which clang++) BUILD_TYPE=[release|debug] LIB_TYPE=[static|shared] $0"
echo ""
echo "Examples:"
echo " CXX=$(which clang++) BUILD_TYPE=release LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=debug LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=release LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which clang++) BUILD_TYPE=debug LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=release LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=debug LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=release LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0"
echo " CXX=$(which g++) BUILD_TYPE=debug LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0"
exit -1
fi
if ${DESTDIR+false}; then
DESTDIR="/usr/local"
fi
# -e: fail on error
# -v: show commands
# -x: show expanded commands
set -vex
env | sort
which python3
which meson
which ninja
echo ${CXX}
${CXX} --version
python3 --version
meson --version
ninja --version
_COMPILER_NAME=`basename ${CXX}`
_BUILD_DIR_NAME="build-${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}"
meson --buildtype ${BUILD_TYPE} --default-library ${LIB_TYPE} . "${_BUILD_DIR_NAME}"
ninja -v -j 2 -C "${_BUILD_DIR_NAME}"
#ninja -v -j 2 -C "${_BUILD_DIR_NAME}" test
cd "${_BUILD_DIR_NAME}"
meson test --no-rebuild --print-errorlogs
if [ "${DESTDIR}" != "/usr/local" ]; then
ninja install
fi
cd -
if ${CLEANUP+false}; then
echo "Skipping cleanup: build directory will persist."
else
rm -r "${_BUILD_DIR_NAME}"
fi

4
.travis_scripts/travis.install.linux.sh
View File

@ -4,10 +4,6 @@ wget https://github.com/ninja-build/ninja/releases/download/v1.7.2/ninja-linux.z
unzip -q ninja-linux.zip -d build
pip3 install meson
# /usr/bin/gcc is 4.6 always, but gcc-X.Y is available.
if [[ $CXX = g++ ]]; then export CXX="g++-4.9" CC="gcc-4.9"; fi
# /usr/bin/clang has a conflict with gcc, so use clang-X.Y.
if [[ $CXX = clang++ ]]; then export CXX="clang++-3.5" CC="clang-3.5"; fi
echo ${PATH}
ls /usr/local
ls /usr/local/bin

34
.travis_scripts/travis.sh
View File

@ -1,34 +0,0 @@
#!/usr/bin/env sh
# This is called by `.travis.yml` via Travis CI.
# Travis supplies $TRAVIS_OS_NAME.
# http://docs.travis-ci.com/user/multi-os/
# Our .travis.yml also defines:
# - SHARED_LIB=ON/OFF
# - STATIC_LIB=ON/OFF
# - CMAKE_PKG=ON/OFF
# - BUILD_TYPE=release/debug
# - VERBOSE_MAKE=false/true
# - VERBOSE (set or not)
# -e: fail on error
# -v: show commands
# -x: show expanded commands
set -vex
env | sort
which python3
which meson
which ninja
echo ${CXX}
${CXX} --version
python3 --version
meson --version
ninja --version
meson --buildtype ${BUILD_TYPE} --default-library ${LIB_TYPE} . build-${LIB_TYPE}
ninja -v -C build-${LIB_TYPE}
#ninja -v -C build-${LIB_TYPE} test
cd build-${LIB_TYPE}
meson test --no-rebuild --print-errorlogs
cd -
rm -r build-${LIB_TYPE}

8
CMakeLists.txt
View File

@ -70,8 +70,6 @@ set( JSONCPP_VERSION ${JSONCPP_VERSION_MAJOR}.${JSONCPP_VERSION_MINOR}.${JSONCPP
message(STATUS "JsonCpp Version: ${JSONCPP_VERSION_MAJOR}.${JSONCPP_VERSION_MINOR}.${JSONCPP_VERSION_PATCH}")
set( JSONCPP_SOVERSION 19 )
enable_testing()
option(JSONCPP_WITH_TESTS "Compile and (for jsoncpp_check) run JsonCpp test executables" ON)
option(JSONCPP_WITH_POST_BUILD_UNITTEST "Automatically run unit-tests as a post build step" ON)
option(JSONCPP_WITH_WARNING_AS_ERROR "Force compilation to fail if a warning occurs" OFF)
@ -175,8 +173,14 @@ if(JSONCPP_WITH_CMAKE_PACKAGE)
FILE jsoncppConfig.cmake)
endif()
if(JSONCPP_WITH_TESTS)
enable_testing()
include(CTest)
endif()
# Build the different applications
add_subdirectory( src )
#install the includes
add_subdirectory( include )

15
CTestConfig.cmake Normal file
View File

@ -0,0 +1,15 @@
## This file should be placed in the root directory of your project.
## Then modify the CMakeLists.txt file in the root directory of your
## project to incorporate the testing dashboard.
##
## # The following are required to submit to the CDash dashboard:
## ENABLE_TESTING()
## INCLUDE(CTest)
set(CTEST_PROJECT_NAME "jsoncpp")
set(CTEST_NIGHTLY_START_TIME "01:23:45 UTC")
set(CTEST_DROP_METHOD "https")
set(CTEST_DROP_SITE "my.cdash.org")
set(CTEST_DROP_LOCATION "/submit.php?project=jsoncpp")
set(CTEST_DROP_SITE_CDASH TRUE)

6
README.md
View File

@ -31,7 +31,11 @@ format to store user input files.
## Contributing to JsonCpp
### Building and testing with Meson/Ninja
Thanks to David Seifert (@SoapGentoo), we (the maintainers) now use [meson](http://mesonbuild.com/) and [ninja](https://ninja-build.org/) to build for debugging, as well as for continuous integration (see [`travis.sh`](travis.sh) ). Other systems may work, but minor things like version strings might break.
Thanks to David Seifert (@SoapGentoo), we (the maintainers) now use
[meson](http://mesonbuild.com/) and [ninja](https://ninja-build.org/) to build
for debugging, as well as for continuous integration (see
[`./travis_scripts/meson_builder.sh`](./travis_scripts/meson_builder.sh) ). Other systems may work, but minor
things like version strings might break.
First, install both meson (which requires Python3) and ninja.
If you wish to install to a directory other than /usr/local, set an environment variable called DESTDIR with the desired path:

6
appveyor.yml
View File

@ -10,7 +10,11 @@ environment:
build_script:
- cmake --version
- cd c:\projects\jsoncpp
- cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX=%CD:\=/%/install -DBUILD_SHARED_LIBS=ON .
- cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX:PATH=%CD:\=/%/install -DBUILD_SHARED_LIBS:BOOL=ON .
# Use ctest to make a dashboard build ctest -D Experimental(Start|Update|Configure|Build|Test|Coverage|MemCheck|Submit)
#NOTE Testing on window is not yet finished - ctest -C Release -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild -D ExperimentalTest -D ExperimentalSubmit
- ctest -C Release -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild -D ExperimentalSubmit
# Final step is to verify that installation succeeds
- cmake --build . --config Release --target install
deploy:

13
src/jsontestrunner/CMakeLists.txt
View File

@ -15,9 +15,22 @@ if(PYTHONINTERP_FOUND)
# Run end to end parser/writer tests
set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../test)
set(RUNJSONTESTS_PATH ${TEST_DIR}/runjsontests.py)
# Run unit tests in post-build
# (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?)
add_custom_target(jsoncpp_readerwriter_tests
"${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
DEPENDS jsontestrunner_exe jsoncpp_test
)
add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests)
## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp
add_test(NAME jsoncpp_readerwriter
COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
WORKING_DIRECTORY "${TEST_DIR}/data"
)
add_test(NAME jsoncpp_readerwriter_json_checker
COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker $<TARGET_FILE:jsontestrunner_exe> "${TEST_DIR}/data"
WORKING_DIRECTORY "${TEST_DIR}/data"
)
endif()

8
src/test_lib_json/CMakeLists.txt
View File

@ -1,6 +1,6 @@
# vim: et ts=4 sts=4 sw=4 tw=0
add_executable( jsoncpp_test
add_executable( jsoncpp_test
jsontest.cpp
jsontest.h
main.cpp
@ -31,6 +31,10 @@ if(JSONCPP_WITH_POST_BUILD_UNITTEST)
POST_BUILD
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:jsoncpp_test>)
endif()
## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp
add_test(NAME jsoncpp_test
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:jsoncpp_test>
)
endif()
set_target_properties(jsoncpp_test PROPERTIES OUTPUT_NAME jsoncpp_test)
set_target_properties(jsoncpp_test PROPERTIES OUTPUT_NAME jsoncpp_test)

40
test/runjsontests.py
View File

@ -55,7 +55,7 @@ def compareOutputs(expected, actual, message):
""" % (message, diff_line,
safeGetLine(expected,diff_line),
safeGetLine(actual,diff_line))
def safeReadFile(path):
try:
return open(path, 'rt', encoding = 'utf-8').read()
@ -69,7 +69,43 @@ def runAllTests(jsontest_executable_path, input_dir = None,
input_dir = os.path.join(os.getcwd(), 'data')
tests = glob(os.path.join(input_dir, '*.json'))
if with_json_checker:
test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
# These tests fail with strict json support, but pass with jsoncpp extra lieniency
"""
Failure details:
* Test ../jsonchecker/fail25.json
Parsing should have failed:
[" tab character in string "]
* Test ../jsonchecker/fail13.json
Parsing should have failed:
{"Numbers cannot have leading zeroes": 013}
* Test ../jsonchecker/fail18.json
Parsing should have failed:
[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]
* Test ../jsonchecker/fail8.json
Parsing should have failed:
["Extra close"]]
* Test ../jsonchecker/fail7.json
Parsing should have failed:
["Comma after the close"],
* Test ../jsonchecker/fail10.json
Parsing should have failed:
{"Extra value after close": true} "misplaced quoted value"
* Test ../jsonchecker/fail27.json
Parsing should have failed:
["line
break"]
"""
known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json",
"fail7.json", "fail10.json", "fail27.json" ]
test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ]
else:
test_jsonchecker = []
failed_tests = []