From 9e23f66f614d23c4c2b540d2d62ce3b2725d422a Mon Sep 17 00:00:00 2001
From: Jordan Bayles <jophba@chromium.org>
Date: Thu, 14 Nov 2019 10:41:25 -0800
Subject: [PATCH] Issue 1102: Fixup test suite, fix broken tests

A recent PR broken the JsonChecker tests by adding support for trailing
commas. This didn't end up breaking the build, because those tests
aren't run, except locally and only using CMake.

This patch fixes the tests by adding exclusions for trailing comma
tests, as well as updates Meson to run these tests as part of `ninja
test`.

See issue #1102.
---
 meson.build          | 10 +++++++++
 test/runjsontests.py | 53 ++++++++++++++------------------------------
 2 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/meson.build b/meson.build
index c293203..e2e92ad 100644
--- a/meson.build
+++ b/meson.build
@@ -105,3 +105,13 @@ test(
     jsontestrunner,
     join_paths(meson.current_source_dir(), 'test/data')]
   )
+test(
+  'jsonchecker_jsontestrunner',
+  python,
+  args : [
+    '-B',
+    join_paths(meson.current_source_dir(), 'test/runjsontests.py'),
+    '--with-json-checker',
+    jsontestrunner,
+    join_paths(meson.current_source_dir(), 'test/data')]
+  )
diff --git a/test/runjsontests.py b/test/runjsontests.py
index 26caf0c..fb1fb39 100644
--- a/test/runjsontests.py
+++ b/test/runjsontests.py
@@ -73,45 +73,26 @@ def runAllTests(jsontest_executable_path, input_dir = None,
         input_dir = os.path.join(os.getcwd(), 'data')
     tests = glob(os.path.join(input_dir, '*.json'))
     if with_json_checker:
-        all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
-        # These tests fail with strict json support, but pass with jsoncpp extra lieniency
-        """
-        Failure details:
-        * Test ../jsonchecker/fail25.json
-        Parsing should have failed:
-        ["	tab	character	in	string	"]
-
-        * Test ../jsonchecker/fail13.json
-        Parsing should have failed:
-        {"Numbers cannot have leading zeroes": 013}
-
-        * Test ../jsonchecker/fail18.json
-        Parsing should have failed:
-        [[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]
-
-        * Test ../jsonchecker/fail8.json
-        Parsing should have failed:
-        ["Extra close"]]
-
-        * Test ../jsonchecker/fail7.json
-        Parsing should have failed:
-        ["Comma after the close"],
-
-        * Test ../jsonchecker/fail10.json
-        Parsing should have failed:
-        {"Extra value after close": true} "misplaced quoted value"
-
-        * Test ../jsonchecker/fail27.json
-        Parsing should have failed:
-        ["line
-        break"]
-        """
-        known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json",
-                                              "fail7.json", "fail10.json", "fail27.json" ]
-        test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ]
+        all_tests = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
+        # These tests fail with strict json support, but pass with JsonCPP's
+        # extra leniency features. When adding a new exclusion to this list,
+        # remember to add the test's number and reasoning here:
+        known = ["fail{}.json".format(n) for n in [
+            4, 9, # fail because we allow trailing commas
+            7,    # fails because we allow commas after close
+            8,    # fails because we allow extra close
+            10,   # fails because we allow extra values after close
+            13,   # fails because we allow leading zeroes in numbers
+            18,   # fails because we allow deeply nested values
+            25,   # fails because we allow tab characters in strings.
+            27,   # fails because we allow string line breaks
+        ]]
+        test_jsonchecker = [ test for test in all_tests
+                             if os.path.basename(test) not in known]
 
     else:
         test_jsonchecker = []
+
     failed_tests = []
     valgrind_path = use_valgrind and VALGRIND_CMD or ''
     for input_path in tests + test_jsonchecker: