testsuite: better reporting of failed tests

git-svn-id: http://caml.inria.fr/svn/ocaml/trunk@13617 f963ae5c-01c2-4b8c-9fe0-0dff7051ff02

testsuite/Makefile

@@ -51,7 +51,8 @@ exec-one:
 	  done; \
         else \
 	  echo "Running tests from '$$DIR' ..."; \
-	  (cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR)); \
+	  cd $(DIR) && \
+	  $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) || echo '=> unexpected error'; \
 	fi
 
 promote: FORCE
@@ -70,13 +71,7 @@ clean: FORCE
 
 report: FORCE
 	@if [ ! -f _log ]; then echo "No '_log' file."; exit 1; fi
-	@echo ''
-	@echo 'Summary:'
-	@echo '  ' `grep 'passed$$' _log | wc -l` 'test(s) passed'
-	@echo '  ' `grep 'failed$$' _log | wc -l` 'test(s) failed'
-	@echo '  ' `grep '^Error' _log | wc -l` 'compilation error(s)'
-	@echo '  ' `grep '^Warning' _log | wc -l` 'compilation warning(s)'
-	@echo '  ' `grep '^make\[2\]: ' _log | wc -l` 'makefile error(s)'
+	@awk -f makefiles/summarize.awk <_log
 
 empty: FORCE
 

testsuite/makefiles/summarize.awk

@@ -0,0 +1,95 @@
+#########################################################################
+#                                                                       #
+#                                 OCaml                                 #
+#                                                                       #
+#         Damien Doligez, projet Gallium, INRIA Rocquencourt            #
+#                                                                       #
+#   Copyright 2013 Institut National de Recherche en Informatique et    #
+#   en Automatique.  All rights reserved.  This file is distributed     #
+#   under the terms of the Q Public License version 1.0.                #
+#                                                                       #
+#########################################################################
+
+function check() {
+    if (!in_test){
+        printf("error at line %d: found test result without test start\n", NR);
+        errored = 1;
+    }
+}
+
+function clear() {
+    curfile = "";
+    in_test = 0;
+}
+
+function record_pass() {
+    check();
+    ++ passed;
+    clear();
+}
+
+function record_fail() {
+    check();
+    ++ failed;
+    fail[failidx++] = sprintf ("%s/%s", curdir, curfile);
+    clear();
+}
+
+function record_unexp() {
+    ++ unexped;
+    unexp[unexpidx++] = sprintf ("%s/%s", curdir, curfile);
+    clear();
+}
+
+/Running tests from '[^']*'/ {
+    if (in_test) record_unexp();
+    match($0, /Running tests from '[^']*'/);
+    curdir = substr($0, RSTART+20, RLENGTH - 21);
+    curfile = "";
+}
+
+/^ ... testing '[^']*'/ {
+    if (in_test) record_unexp();
+    match($0, /... testing '[^']*'/);
+    curfile = substr($0, RSTART+13, RLENGTH-14);
+    in_test = 1;
+}
+
+/^ ... testing with / {
+    if (in_test) record_unexp();
+    in_test = 1;
+}
+
+/=> passed/ {
+    record_pass();
+}
+
+/=> failed/ {
+    record_fail();
+}
+
+/=> unexpected error/ {
+    record_unexp();
+}
+
+END {
+    if (errored){
+        exit (3);
+    }else{
+        printf("\n");
+        printf("Summary:\n");
+        printf("  %3d test(s) passed\n", passed);
+        printf("  %3d test(s) failed\n", failed);
+        printf("  %3d unexpected error(s)\n", unexped);
+        if (failed != 0){
+            printf("\nList of failed tests:\n");
+            for (i in fail) printf("    %s\n", fail[i]);
+        }
+        if (unexped != 0){
+            printf("\nList of unexpected errors:\n");
+            for (i in unexp) printf("    %s\n", unexp[i]);
+        }
+        printf("\n");
+        exit (failed || unexped ? 4 : 0);
+    }
+}