ocaml/testsuite/summarize.awk

232 lines
6.8 KiB
Awk
Raw Normal View History

#**************************************************************************
#* *
#* OCaml *
#* *
#* Damien Doligez, projet Gallium, INRIA Rocquencourt *
#* *
#* Copyright 2013 Institut National de Recherche en Informatique et *
#* en Automatique. *
#* *
#* All rights reserved. This file is distributed under the terms of *
#* the GNU Lesser General Public License version 2.1, with the *
#* special exception on linking described in the file LICENSE. *
#* *
#**************************************************************************
function check() {
if (!in_test){
printf("error at line %d: found test result without test start\n", NR);
errored = 1;
}
}
function clear() {
curfile = "";
in_test = 0;
}
function record_pass() {
check();
if (!(key in RESULTS)) ++nresults;
RESULTS[key] = "p";
delete SKIPPED[curdir];
clear();
}
function record_skip() {
check();
if (!(key in RESULTS)) ++nresults;
RESULTS[key] = "s";
if (curdir in SKIPPED) SKIPPED[curdir] = 1;
clear();
}
function record_na() {
check();
if (!(key in RESULTS)) ++nresults;
RESULTS[key] = "n";
if (curdir in SKIPPED) SKIPPED[curdir] = 1;
clear();
}
# The output cares only if the test passes at least once so if a test passes,
# but then fails in a re-run triggered by a different test, ignore it.
function record_fail() {
check();
if (!(key in RESULTS) || RESULTS[key] == "s"){
if (!(key in RESULTS)) ++nresults;
RESULTS[key] = "f";
}
delete SKIPPED[curdir];
clear();
}
function record_unexp() {
if (!(key in RESULTS) || RESULTS[key] == "s"){
if (!(key in RESULTS)) ++nresults;
RESULTS[key] = "e";
}
delete SKIPPED[curdir];
clear();
}
2020-06-23 11:47:38 -07:00
/^> / {
next;
}
/Running tests from '[^']*'/ {
if (in_test) record_unexp();
match($0, /Running tests from '[^']*'/);
curdir = substr($0, RSTART+20, RLENGTH - 21);
# Use SKIPPED[curdir] as a sentinel to detect no output
SKIPPED[curdir] = 0;
key = curdir;
DIRS[key] = key;
curfile = "";
}
/ ... testing.* ... testing/ {
printf("error at line %d: found two test results on the same line\n", NR);
errored = 1;
}
/^ ... testing '[^']*'/ {
if (in_test) record_unexp();
match($0, /... testing '[^']*'/);
curfile = substr($0, RSTART+13, RLENGTH-14);
if (match($0, /... testing '[^']*' with [^:=]*/)){
curfile = substr($0, RSTART+12, RLENGTH-12);
}
key = sprintf ("%s/%s", curdir, curfile);
DIRS[key] = curdir;
in_test = 1;
}
/^ ... testing (with|[^'])/ {
if (in_test) record_unexp();
key = curdir;
DIRS[key] = curdir;
in_test = 1;
}
/=> passed/ {
record_pass();
}
/=> skipped/ {
record_skip();
}
/=> n\/a/ {
record_na();
}
/=> failed/ {
record_fail();
}
/=> unexpected error/ {
record_unexp();
}
/^re-ran / {
if (in_test){
printf("error at line %d: found re-ran inside a test\n", NR);
errored = 1;
}else{
RERAN[substr($0, 8, length($0)-7)] += 1;
++ reran;
}
}
END {
if (errored){
printf ("\n#### Some fatal error occurred during testing.\n\n");
exit (3);
}else{
if (!retries){
for (key in SKIPPED){
if (!SKIPPED[key]){
++ empty;
blanks[emptyidx++] = key;
delete SKIPPED[key];
}
}
for (key in RESULTS){
r = RESULTS[key];
if (r == "p"){
++ passed;
}else if (r == "f"){
++ failed;
fail[failidx++] = key;
}else if (r == "e"){
++ unexped;
unexp[unexpidx++] = key;
}else if (r == "s"){
++ skipped;
curdir = DIRS[key];
if (curdir in SKIPPED){
if (SKIPPED[curdir]){
SKIPPED[curdir] = 0;
skips[skipidx++] = curdir;
}
}else{
skips[skipidx++] = key;
}
}else if (r == "n"){
++ ignored;
}
}
printf("\n");
if (skipped != 0){
printf("\nList of skipped tests:\n");
for (i=0; i < skipidx; i++) printf(" %s\n", skips[i]);
}
if (empty != 0){
printf("\nList of directories returning no results:\n");
for (i=0; i < empty; i++) printf(" %s\n", blanks[i]);
}
if (failed != 0){
printf("\nList of failed tests:\n");
for (i=0; i < failed; i++) printf(" %s\n", fail[i]);
}
if (unexped != 0){
printf("\nList of unexpected errors:\n");
for (i=0; i < unexped; i++) printf(" %s\n", unexp[i]);
}
printf("\n");
printf("Summary:\n");
printf(" %3d tests passed\n", passed);
printf(" %3d tests skipped\n", skipped);
printf(" %3d tests failed\n", failed);
printf(" %3d tests not started (parent test skipped or failed)\n",
ignored);
printf(" %3d unexpected errors\n", unexped);
printf(" %3d tests considered", nresults);
2018-06-30 03:14:29 -07:00
if (nresults != passed + skipped + ignored + failed + unexped){
printf (" (totals don't add up??)");
}
2018-06-30 03:14:29 -07:00
printf ("\n");
if (reran != 0){
printf(" %3d test dir re-runs\n", reran);
}
if (failed || unexped){
printf("#### Something failed. Exiting with error status.\n\n");
exit 4;
}
}else{
for (key in RESULTS){
if (RESULTS[key] == "f" || RESULTS[key] == "e"){
key = DIRS[key];
if (!(key in RERUNS)){
RERUNS[key] = 1;
if (RERAN[key] < max_retries){
printf("%s\n", key);
}
}
}
}
}
}
}