feat(tests): enhance test framework for minetest mods

- Enhance instructions in the README for setting up and running tests with the test framework.
- Develop modules for registering and running tests within mods.
- Provide a mechanism to control which mods are tested through `test_harness_mods` setting.
- Improve summary display logic for test results and categorize them (passed, failed, skipped, dnr).
- Refactor scripts for better management of test execution and handling various cases.

.util/Dockerfile

@@ -1,5 +1,6 @@
 FROM curlimages/curl:latest as dl_base
 
+# list of mods and game to download
 RUN mkdir -p minetest_game && curl -s -L https://github.com/minetest/minetest_game/archive/refs/heads/master.tar.gz | tar zxvf - -C minetest_game --strip-components=1
 
 FROM ghcr.io/minetest/minetest:latest as builder
@@ -17,6 +18,7 @@ RUN mkdir -p /usr/local/share/minetest/games && \
 
 # WORKDIR /config/.minetest/games/devtest/mods
 
+# Copy from the dl base the game and mods to their correct path in the game or mod folders
 COPY --from=dl_base /home/curl_user/minetest_game /usr/local/share/minetest/games/minetest
 
 COPY <<"EOF" /usr/local/sbin/run_minetest
@@ -62,9 +64,10 @@ COPY <<-EOF /var/lib/minetest/.minetest/world/map_meta.txt
     [end_of_params]
 EOF
 
+# Customize the test_harness_mods value to list the mods to test. Empty value means all, including the test harness itself
 COPY <<-EOF /etc/minetest/minetest.conf.base
+    test_harness_mods=test_harness
     test_harness_run_tests=true
-    test_harness_run_internal_tests=true
     max_forceloaded_blocks=9999
     name=admin
     creative_mode=true

README.md

@@ -12,15 +12,47 @@ This mod provides an automated testing framework for Minetest mods by simulating
 - **Area and Node Manipulation**: Test areas can be defined, and node checks can be performed.
 - **Test Context and State Management**: Manage and track the progress of tests across multiple mods.
 
-## Installation
+## Prerequisite
 
-1. Clone or download the `docker-test-harness` mod.
+The tests are run using Docker or Podman. Therefor you'll need to have either one of them installed.
-2. Place the `test_harness` directory into the `mods` folder of your Minetest world.
+
-3. Add the following setting in `minetest.conf` to enable automatic test execution:
+## Usage
+
+This mod's goal is to provide a test framework for implementing tests for minetest mods.
+
+1. Declare `test_harness` as an optional dependencies for your mod
+2. Clone or download the `docker-test-harness` mod.
+3. Place the `test_harness` directory into the `mods` folder of your Minetest world.
+4. copy the `.util` folder in your project
+5. Customize the `.util/Dockerfile` to include the mods and game your project depends on (download and copy)
+6. Customize the `.util/Dockerfile` to adapt the `/etc/minetest/minetest.conf.base` file to list the mods you want to test (`test_harness_mods`), and to adjust the server configuration to your project test settings.
+7. Do not forget the following setting in `/etc/minetest/minetest.conf.base` to enable automatic test execution:/etc/minetest/minetest.conf.base
     ```ini
     test_harness_run_tests = true
     ```
-    
+8. In your source files (in the `init.lua` for example) add the following lines (or equivalent)
+    ```lua
+    if minetest.settings:get_bool("test_harness_run_tests", false) then
+	    dofile(minetest.get_modpath("my_mod").. "/test/init.lua")
+    end
+    ```
+9. In your test file(s), get an instance of the method allowing the registration of your tests:
+   ```lua
+   local register_test = test_harness.get_test_registrator("my_mod", my_mod.version_string)
+   ```
+10. Register the test by calling `register_test`
+11. Run the test by lauching the `run_tests.sh` script. For example with Podman
+    ```shell
+    $ .util/run_tests.sh --podman
+    ```
+    See the available options with `.util/run_tests.sh --help`
+
+## Sources
+
+- The base pf the code comes from the WorldEdit mod project : https://github.com/Uberi/Minetest-WorldEdit.
+- The client's Dockerfile has been adapted from `Miney-pi` (https://github.com/miney-py/minetest-client-docker)
+- The color management code comes from `lua-chroma`: https://github.com/ldrumm/lua-chroma
+
 ## License
 
 Licensed under the AGPL (Affero General Public License).

base.lua

@@ -11,10 +11,18 @@ local tests_state = TESTS_STATE_ENUM.NOT_STARTED
 
 local tests_context = {}
 
+local tests_mod_list = {}
+
+local test_mods_str = minetest.settings:get("test_harness_mods") or ""
+for str in string.gmatch(test_mods_str, "[^,]+") do
+    str = str:gsub("%s+", "")
+    tests_mod_list[str] = true
+end
+
 local failed = 0
 
-test_harness.set_context = function(mod, version_string)
+local set_context = function(mod, version_string)
-    table.insert(tests_context, { mod = mod, version_string = version_string})
+    tests_context[mod] = { mod = mod, version_string = version_string}
 end
 
 local register_test = function(mod, name, func, opts)
@@ -43,7 +51,7 @@ local register_test = function(mod, name, func, opts)
     table.insert(mod_test_list, opts)
 end
 
-test_harness.get_test_registrator = function(mod)
+test_harness.get_test_registrator = function(mod, version_string)
     local modnames = minetest.get_modnames()
     local is_mod = false
     for i=1,#modnames do
@@ -53,8 +61,13 @@ test_harness.get_test_registrator = function(mod)
         end
     end
     if not is_mod then error("get_test_registrator given mod "..mod.." is not a mod.") end
-    return function(name, func, opts)
+    if #tests_mod_list == 0 or tests_mod_list[mod] then
-        register_test(mod, name, func, opts)
+        set_context(mod, version_string)
+        return function(name, func, opts)
+            register_test(mod, name, func, opts)
+        end
+    else
+        return function() end
     end
 end
 
@@ -373,31 +386,37 @@ rawset(_G, "pprint", setmetatable({
     },
     _sequence = '',
     _highlight = false,
-    print = io.write
 },
 {
-    __call = function(self, ...) return io.write(...) end,
+    __call = function(self, ...)
+        local arg = {...}
+        local add_sequence = (not not self._sequence) and #self._sequence > 0
+        local call_res = (add_sequence and self._sequence or '')
+
+        for _,v in ipairs(arg) do
+            call_res = call_res .. v
+        end
+
+        self._sequence = ''
+
+        return call_res .. ((add_sequence and rawget(self, "escapes").clear) or '')
+
+     end,
 
     __index = function(self, index)
+
         local esc = self._highlight and rawget(self, 'escapes').highlight[index]
             or rawget(self, 'escapes')[index]
         self._highlight = index == 'highlight'
         if esc ~= nil then
             if type(esc) == 'string' then
-                self._sequence = self._sequence .. esc
+                if index == 'clear' then
+                    self._sequence = ""
+                else
+                    self._sequence = self._sequence .. esc
+                end
             end
-            return setmetatable({}, {
+            return self
-                __call = function(proxy, ...)
-                    if self._sequence then io.write(self._sequence) end
-                    self.print(...)
-                    self._sequence = ''
-                    io.write(rawget(self,'escapes').clear)
-                    return self
-                end,
-                __index = function(proxy, k)
-                    return self[k]
-                end,
-            })
         else
             return rawget(self, index)
         end
@@ -428,51 +447,73 @@ end
 local print_result_line = function(test)
     local s = ":"..test.mod..":"
     local rest = s .. test.name
-    pprint.light_gray(s)
+    print(string.format("%s %s%s%s",
-    pprint(" ")
+        pprint.light_gray(s),
-    pprint(test.name)
+        test.name,
-    pprint(string.rep(" ", 80 - #rest))
+        string.rep(" ", 80 - #rest),
-    if test.result.ok then pprint.green("pass") else pprint.red("FAIL") end
+        test.result.ok and pprint.green("pass") or pprint.red("FAIL")
-    pprint("\n")
+    ))
     if not test.result.ok and test.result.err then
-        pprint.yellow("   " .. test.result.err .. "\n")
+        print(pprint.yellow("   " .. test.result.err))
     end
 end
 
 local display_tests_summary = function()
 
+    local title = "TESTS RUN SUMMARY"
+    local remaining_width = 72 - #title
+    local left = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width-1)/2
+    local right = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width+1)/2
     print(string.rep("-",80))
     print("----"..string.rep(" ",72).."----")
-    pprint("----"..string.rep(" ",27))
+    print(string.format("----%s%s%s----",
-    pprint.bold.underline.orange("TESTS RUN SUMMARY")
+        string.rep(" ",left),
-    print(string.rep(" ",28).."----")
+        pprint.bold.underline.orange(title),
+        string.rep(" ",right)
+    ))
     print("----"..string.rep(" ",72).."----")
     print(string.rep("-",80))
 
     print("All tests done, " .. failed .. " tests failed.")
     print()
 
-    for mod, tests_list in pairs(tests_by_mod) do
+    local test_counters = { total=0, passed=0, failed=0, skipped=0, dnr=0}
-        pprint.baby_blue(string.format("%#80s\n", mod))
+
+    for mod, tests_list in pairs(tests_by_mod) do 
+        print(pprint.baby_blue(string.format("%#80s", mod)))
         for _, test in ipairs(tests_list) do
             if test.func == nil then
                 local s = ":".. test.mod ..":---- " .. test.name
-                pprint.light_gray(":".. test.mod ..":").blue("---- " .. test.name)
+                print(pprint.light_gray(":".. test.mod ..":")..pprint.blue("---- " .. test.name .. string.rep("-", 80 - #s)))
-                pprint.blue(string.rep("-", 80 - #s).."\n")
             elseif test.result ~= nil then
+                test_counters["total"] = test_counters["total"] + 1
+                if test.result.ok == nil then
+                    test_counters["skipped"] = test_counters["skipped"] + 1
+                elseif test.result.ok then
+                    test_counters["passed"] = test_counters["passed"] + 1
+                else
+                    test_counters["failed"] = test_counters["failed"] + 1
+                end
                 print_result_line(test)
             else
+                test_counters["total"] = test_counters["total"] + 1
+                test_counters["dnr"] = test_counters["dnr"] + 1
                 local s = ":"..test.mod..":"
                 local rest = s .. test.name
-                pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."dnr\n")
+                print(pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."dnr"))
             end
         end
-        pprint.baby_blue(string.rep("-",80),"\n")
+        print(pprint.baby_blue(string.rep("-",80)))
     end
     print(string.rep("-",80))
-    pprint.bold("Tests done, ")
+    print(string.format("%s%s%s",
-    if failed == 0 then pprint.bold.green(failed) else pprint.bold.red(failed) end
+        pprint.bold(string.format("%d Tests done, ", test_counters.total)),
-    pprint.bold(" tests failed.\n")
+        (test_counters.failed==0 and pprint.bold.green(test_counters.failed)) or pprint.bold.red(test_counters.failed),
+        pprint.bold(" test(s) failed,")
+    ))
+    print(string.format("%d test(s) passed,", test_counters.passed))
+    print(string.format("%d test(s) skipped,", test_counters.skipped))
+    print(string.format("%d test(s) dnr.", test_counters.dnr))
     print(string.rep("-",80))
 end
 
@@ -591,15 +632,16 @@ local run_tests = function()
         local current_title = {}
         for _, test in ipairs(tests) do
             if not test.func then
-                nb_tests = nb_tests + 1
                 table.insert(current_title, test)
             elseif test.players and next(test.players) then
+                nb_tests = nb_tests + 1
                 for _, t in ipairs(current_title) do
                     table.insert(players_tests, t)
                 end
                 current_title = {}
                 table.insert(players_tests, test)
             else
+                nb_tests = nb_tests + 1
                 for _, t in ipairs(current_title) do
                     table.insert(simple_tests, t)
                 end
@@ -611,8 +653,8 @@ local run_tests = function()
 
 
         print("Running " .. nb_tests .. " tests for:")
-        for _,context in ipairs(tests_context) do
+        for _,context in pairs(tests_context) do
-            print("  - "..context.mod .." - "..context.version_string)
+            print("  - "..context.mod .." - "..(context.version_string or ""))
         end
         print("on " .. v.project .. " " .. (v.hash or v.string))
     end
@@ -659,10 +701,10 @@ local run_tests = function()
         end
         if tests_state == TESTS_STATE_ENUM.DONE then
             display_tests_summary()
-        
             if minetest.settings:get_bool("test_harness_stop_server", true) then
                 request_shutdown()
             end
+            return
         end
 
         -- list of needed players

init.lua

@@ -8,9 +8,5 @@ test_harness.version_string = string.format("%d.%d", ver.major, ver.minor)
 
 if minetest.settings:get_bool("test_harness_run_tests", false) then
 	dofile(test_harness.modpath .. "/base.lua")
-end
+	dofile(test_harness.modpath .. "/test/init.lua")
-
+end
-if minetest.settings:get_bool("test_harness_run_internal_tests", false) then
-	test_harness.set_context("test_harness", test_harness.version_string)
-    dofile(test_harness.modpath .. "/test/init.lua")
-end

settingtypes.txt

@@ -1,5 +1,5 @@
 test_harness_run_tests (Run registered tests) bool false
-test_harness_run_internal_tests (Run registered tests) bool false
+test_harness_mods (Comma separated list of mod to run the tests for. Empty means all mods) string ""
 test_harness_failfast (Stops at the first error) bool false
 test_harness_stop_server (Stop the server after the end of tests) bool true
-test_harness_test_players_password (Password for the test players) string test
+test_harness_test_players_password (Password for the test players) string test

test/init.lua

@@ -1,4 +1,4 @@
-local register_test = test_harness.get_test_registrator("test_harness")
+local register_test = test_harness.get_test_registrator("test_harness", test_harness.version_string)
 local get_node = minetest.get_node
 local set_node = minetest.set_node
 local air = "air"

test/players.lua

@@ -1,4 +1,4 @@
-local register_test = test_harness.get_test_registrator("test_harness")
+local register_test = test_harness.get_test_registrator("test_harness", test_harness.version_string)
 local vec = vector.new
 
 register_test("Tests utilities for players")