refactor(test_harness): improve test handling and display

Refactored the `run_player_tests` function in `base.lua` to improve error handling and the way test results are displayed. Tests now run asynchronously and reschedule remaining tests if necessary. Added clearer output for skipped tests and improved failure reporting.

.util/run_tests.sh

@@ -25,7 +25,7 @@ EOF
 
 cleanup() {
   trap - SIGINT SIGTERM ERR EXIT
-  # script cleanup here
+  eval $docker_cmd compose -f "${script_dir}/docker-compose.yaml" down
 }
 
 setup_colors() {
@@ -200,6 +200,4 @@ eval $docker_cmd compose -f "${script_dir}/docker-compose.yaml" build --pull
 eval $docker_cmd compose -f "${script_dir}/docker-compose.yaml" up --force-recreate --exit-code-from server --abort-on-container-exit
 exit_code=$?
 
-eval $docker_cmd compose -f "${script_dir}/docker-compose.yaml" down
-
 exit $exit_code

base.lua

@@ -446,7 +446,9 @@ local display_tests_summary = function()
                     pprint.yellow("   " .. test.result.err .. "\n")
                 end
             else
-                pprint.light_gray(string.format(":%s:%-80s %s\n", test.mod, test.name, "No result"))
+                local s = ":"..test.mod..":"
+                local rest = s .. test.name
+                pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."skip\n")
             end
         end
         pprint.baby_blue(string.rep("-",80),"\n")
@@ -505,13 +507,7 @@ end
 
 
 local set_tests_done = function()
-    tests_state = TESTS_STATE_ENUM.DONE
-    print("All tests done, " .. failed .. " tests failed.")
-    display_tests_summary()
 
-    if minetest.settings:get_bool("test_harness_stop_server", true) then
-        request_shutdown()
-    end
 end
 
 local get_connected_player_names = function()
@@ -522,59 +518,56 @@ local get_connected_player_names = function()
     return connected_player_names
 end
 
-local run_player_tests = function(list_player_tests)
+test_harness.run_player_tests = function(list_player_tests, area)
-    if tests_state == TESTS_STATE_ENUM.DONE then
+
-        return
-    end
     local connected_player_names = get_connected_player_names()
-
-    local test_run_callback = function()
-        for _, test in ipairs(list_player_tests) do
-            if test.func ~= nil and test.result == nil then return end
-        end
-        set_tests_done()
-    end
-
     for _, test in ipairs(list_player_tests) do
-        if tests_state == TESTS_STATE_ENUM.DONE then
-            break
-        end
         if not test.result and
           test.func and
           test.players and
           next(test.players) and
           all_in_table(test.players, connected_player_names)
         then
-            local wanted = vec(56, 56, 56)
+            local player_data = test_harness.save_players(test.players)
-            for x = 0, math.floor(wanted.x / 16) do
+            area.clear()
-                for y = 0, math.floor(wanted.y / 16) do
+            local ok, err = pcall(test.func)
-                    for z = 0, math.floor(wanted.z / 16) do
+            test.result = { ok = ok, err = err }
-                        assert(minetest.forceload_block(vec(x * 16, y * 16, z * 16), true, -1))
+            print(string.format(":%s:%-60s %s", test.mod, test.name, ok and "pass" or "FAIL"))
-                    end
+            test_harness.restore_players(player_data)
+            if not ok then
+                print("   " .. err)
+                failed = failed + 1
+                if minetest.settings:get_bool("test_harness_failfast", false) then
+                    tests_state = TESTS_STATE_ENUM.DONE
+                    break
                 end
             end
-            area.assign(vec(0, 0, 0), wanted, function()
-                area.clear()
-                local player_data = test_harness.save_players(test.players)
-                local ok, err = pcall(test.func)
-                test.result = { ok = ok, err = err }
-                print(string.format(":%s:%-60s %s", test.mod, test.name, ok and "pass" or "FAIL"))
-                if not ok then
-                    print("   " .. err)
-                    failed = failed + 1
-                    if minetest.settings:get_bool("test_harness_failfast", false) then
-                        if minetest.settings:get_bool("test_harness_stop_server", true) then
-                            request_shutdown()
-                        else
-                            set_tests_done()
-                        end
-                    end
-                end
-                test_harness.restore_players(player_data)
-                test_run_callback()
-            end)
         end
     end
+
+    local remaining_tests = {}
+    if tests_state ~= TESTS_STATE_ENUM.DONE then
+        for _, test in ipairs(list_player_tests) do
+            if test.func ~= nil and test.result == nil then
+                table.insert(remaining_tests,test)
+            end
+        end
+        if #remaining_tests == 0 then
+            tests_state = TESTS_STATE_ENUM.DONE
+        end
+    end
+
+    if tests_state == TESTS_STATE_ENUM.DONE then
+        print("All tests done, " .. failed .. " tests failed.")
+        display_tests_summary()
+
+        if minetest.settings:get_bool("test_harness_stop_server", true) then
+            request_shutdown()
+        end
+    else
+        -- reschedule
+        minetest.after(1,test_harness.run_player_tests,remaining_tests, area)
+    end
 end
 
 
@@ -630,6 +623,7 @@ local run_tests = function()
     end
     failed = 0
     area.assign(vec(0, 0, 0), wanted, function()
+        -- run the simple tests
         for _, test in ipairs(simple_tests) do
             if not test.func then
                 local s = ":"..test.mod..":---- " .. test.name .. " "
@@ -678,26 +672,7 @@ local run_tests = function()
         end
 
         -- launch test
-        minetest.register_on_joinplayer(function(player)
+        test_harness.run_player_tests(players_tests, area)
-            -- if player tests are started or done, do nothing
-            if tests_state == TESTS_STATE_ENUM.DONE or tests_state == TESTS_STATE_ENUM.STARTED_PLAYERS then return end
-
-            -- else check that all necessary players are connected before starting the tests (we have the list in players_table)
-            local connected_player_names = get_connected_player_names()
-            if all_in_table(needed_playernames, connected_player_names) then
-                tests_state = TESTS_STATE_ENUM.STARTED_PLAYERS
-                run_player_tests(players_tests)
-            end
-        end)
-
-        minetest.register_on_leaveplayer(function(player, timeout)
-            if tests_state ~= TESTS_STATE_ENUM.STARTED_PLAYERS then return end
-            -- check that needed playernames is still in the list of connected players
-            local connected_player_names = get_connected_player_names()
-            if not all_in_table(needed_playernames, connected_player_names) then
-                tests_state = TESTS_STATE_ENUM.STARTED
-            end
-        end)
     end)
 end