minetest/docker_test_harness-cd2025
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tests): enhance test framework for minetest mods

Browse Source 
- Enhance instructions in the README for setting up and running tests with the test framework.
- Develop modules for registering and running tests within mods.
- Provide a mechanism to control which mods are tested through `test_harness_mods` setting.
- Improve summary display logic for test results and categorize them (passed, failed, skipped, dnr).
- Refactor scripts for better management of test execution and handling various cases.
This commit is contained in:
Yves-Marie Haussonne 2024-10-06 09:19:43 +02:00
parent b322a1b6b1
commit 8de4cc3964
7 changed files with 132 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.util/Dockerfile
View File

@ -1,5 +1,6 @@
FROM curlimages/curl:latest as dl_base
# list of mods and game to download
RUN mkdir -p minetest_game && curl -s -L https://github.com/minetest/minetest_game/archive/refs/heads/master.tar.gz | tar zxvf - -C minetest_game --strip-components=1
FROM ghcr.io/minetest/minetest:latest as builder
@ -17,6 +18,7 @@ RUN mkdir -p /usr/local/share/minetest/games && \
# WORKDIR /config/.minetest/games/devtest/mods
# Copy from the dl base the game and mods to their correct path in the game or mod folders
COPY --from=dl_base /home/curl_user/minetest_game /usr/local/share/minetest/games/minetest
COPY <<"EOF" /usr/local/sbin/run_minetest
@ -62,9 +64,10 @@ COPY <<-EOF /var/lib/minetest/.minetest/world/map_meta.txt
[end_of_params]
EOF
# Customize the test_harness_mods value to list the mods to test. Empty value means all, including the test harness itself
COPY <<-EOF /etc/minetest/minetest.conf.base
test_harness_mods=test_harness
test_harness_run_tests=true
test_harness_run_internal_tests=true
max_forceloaded_blocks=9999
name=admin
creative_mode=true

40
README.md
View File

@ -12,14 +12,46 @@ This mod provides an automated testing framework for Minetest mods by simulating
- **Area and Node Manipulation**: Test areas can be defined, and node checks can be performed.
- **Test Context and State Management**: Manage and track the progress of tests across multiple mods.
## Installation
## Prerequisite
1. Clone or download the `docker-test-harness` mod.
2. Place the `test_harness` directory into the `mods` folder of your Minetest world.
3. Add the following setting in `minetest.conf` to enable automatic test execution:
The tests are run using Docker or Podman. Therefor you'll need to have either one of them installed.
## Usage
This mod's goal is to provide a test framework for implementing tests for minetest mods.
1. Declare `test_harness` as an optional dependencies for your mod
2. Clone or download the `docker-test-harness` mod.
3. Place the `test_harness` directory into the `mods` folder of your Minetest world.
4. copy the `.util` folder in your project
5. Customize the `.util/Dockerfile` to include the mods and game your project depends on (download and copy)
6. Customize the `.util/Dockerfile` to adapt the `/etc/minetest/minetest.conf.base` file to list the mods you want to test (`test_harness_mods`), and to adjust the server configuration to your project test settings.
7. Do not forget the following setting in `/etc/minetest/minetest.conf.base` to enable automatic test execution:/etc/minetest/minetest.conf.base
```ini
test_harness_run_tests = true
```
8. In your source files (in the `init.lua` for example) add the following lines (or equivalent)
```lua
if minetest.settings:get_bool("test_harness_run_tests", false) then
dofile(minetest.get_modpath("my_mod").. "/test/init.lua")
end
```
9. In your test file(s), get an instance of the method allowing the registration of your tests:
```lua
local register_test = test_harness.get_test_registrator("my_mod", my_mod.version_string)
```
10. Register the test by calling `register_test`
11. Run the test by lauching the `run_tests.sh` script. For example with Podman
```shell
$ .util/run_tests.sh --podman
```
See the available options with `.util/run_tests.sh --help`
## Sources
- The base pf the code comes from the WorldEdit mod project : https://github.com/Uberi/Minetest-WorldEdit.
- The client's Dockerfile has been adapted from `Miney-pi` (https://github.com/miney-py/minetest-client-docker)
- The color management code comes from `lua-chroma`: https://github.com/ldrumm/lua-chroma
## License

126
base.lua
View File

@ -11,10 +11,18 @@ local tests_state = TESTS_STATE_ENUM.NOT_STARTED
local tests_context = {}
local tests_mod_list = {}
local test_mods_str = minetest.settings:get("test_harness_mods") or ""
for str in string.gmatch(test_mods_str, "[^,]+") do
str = str:gsub("%s+", "")
tests_mod_list[str] = true
end
local failed = 0
test_harness.set_context = function(mod, version_string)
table.insert(tests_context, { mod = mod, version_string = version_string})
local set_context = function(mod, version_string)
tests_context[mod] = { mod = mod, version_string = version_string}
end
local register_test = function(mod, name, func, opts)
@ -43,7 +51,7 @@ local register_test = function(mod, name, func, opts)
table.insert(mod_test_list, opts)
end
test_harness.get_test_registrator = function(mod)
test_harness.get_test_registrator = function(mod, version_string)
local modnames = minetest.get_modnames()
local is_mod = false
for i=1,#modnames do
@ -53,8 +61,13 @@ test_harness.get_test_registrator = function(mod)
end
end
if not is_mod then error("get_test_registrator given mod "..mod.." is not a mod.") end
return function(name, func, opts)
register_test(mod, name, func, opts)
if #tests_mod_list == 0 or tests_mod_list[mod] then
set_context(mod, version_string)
return function(name, func, opts)
register_test(mod, name, func, opts)
end
else
return function() end
end
end
@ -373,31 +386,37 @@ rawset(_G, "pprint", setmetatable({
},
_sequence = '',
_highlight = false,
print = io.write
},
{
__call = function(self, ...) return io.write(...) end,
__call = function(self, ...)
local arg = {...}
local add_sequence = (not not self._sequence) and #self._sequence > 0
local call_res = (add_sequence and self._sequence or '')
for _,v in ipairs(arg) do
call_res = call_res .. v
end
self._sequence = ''
return call_res .. ((add_sequence and rawget(self, "escapes").clear) or '')
end,
__index = function(self, index)
local esc = self._highlight and rawget(self, 'escapes').highlight[index]
or rawget(self, 'escapes')[index]
self._highlight = index == 'highlight'
if esc ~= nil then
if type(esc) == 'string' then
self._sequence = self._sequence .. esc
if index == 'clear' then
self._sequence = ""
else
self._sequence = self._sequence .. esc
end
end
return setmetatable({}, {
__call = function(proxy, ...)
if self._sequence then io.write(self._sequence) end
self.print(...)
self._sequence = ''
io.write(rawget(self,'escapes').clear)
return self
end,
__index = function(proxy, k)
return self[k]
end,
})
return self
else
return rawget(self, index)
end
@ -428,51 +447,73 @@ end
local print_result_line = function(test)
local s = ":"..test.mod..":"
local rest = s .. test.name
pprint.light_gray(s)
pprint(" ")
pprint(test.name)
pprint(string.rep(" ", 80 - #rest))
if test.result.ok then pprint.green("pass") else pprint.red("FAIL") end
pprint("\n")
print(string.format("%s %s%s%s",
pprint.light_gray(s),
test.name,
string.rep(" ", 80 - #rest),
test.result.ok and pprint.green("pass") or pprint.red("FAIL")
))
if not test.result.ok and test.result.err then
pprint.yellow(" " .. test.result.err .. "\n")
print(pprint.yellow(" " .. test.result.err))
end
end
local display_tests_summary = function()
local title = "TESTS RUN SUMMARY"
local remaining_width = 72 - #title
local left = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width-1)/2
local right = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width+1)/2
print(string.rep("-",80))
print("----"..string.rep(" ",72).."----")
pprint("----"..string.rep(" ",27))
pprint.bold.underline.orange("TESTS RUN SUMMARY")
print(string.rep(" ",28).."----")
print(string.format("----%s%s%s----",
string.rep(" ",left),
pprint.bold.underline.orange(title),
string.rep(" ",right)
))
print("----"..string.rep(" ",72).."----")
print(string.rep("-",80))
print("All tests done, " .. failed .. " tests failed.")
print()
local test_counters = { total=0, passed=0, failed=0, skipped=0, dnr=0}
for mod, tests_list in pairs(tests_by_mod) do
pprint.baby_blue(string.format("%#80s\n", mod))
print(pprint.baby_blue(string.format("%#80s", mod)))
for _, test in ipairs(tests_list) do
if test.func == nil then
local s = ":".. test.mod ..":---- " .. test.name
pprint.light_gray(":".. test.mod ..":").blue("---- " .. test.name)
pprint.blue(string.rep("-", 80 - #s).."\n")
print(pprint.light_gray(":".. test.mod ..":")..pprint.blue("---- " .. test.name .. string.rep("-", 80 - #s)))
elseif test.result ~= nil then
test_counters["total"] = test_counters["total"] + 1
if test.result.ok == nil then
test_counters["skipped"] = test_counters["skipped"] + 1
elseif test.result.ok then
test_counters["passed"] = test_counters["passed"] + 1
else
test_counters["failed"] = test_counters["failed"] + 1
end
print_result_line(test)
else
test_counters["total"] = test_counters["total"] + 1
test_counters["dnr"] = test_counters["dnr"] + 1
local s = ":"..test.mod..":"
local rest = s .. test.name
pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."dnr\n")
print(pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."dnr"))
end
end
pprint.baby_blue(string.rep("-",80),"\n")
print(pprint.baby_blue(string.rep("-",80)))
end
print(string.rep("-",80))
pprint.bold("Tests done, ")
if failed == 0 then pprint.bold.green(failed) else pprint.bold.red(failed) end
pprint.bold(" tests failed.\n")
print(string.format("%s%s%s",
pprint.bold(string.format("%d Tests done, ", test_counters.total)),
(test_counters.failed==0 and pprint.bold.green(test_counters.failed)) or pprint.bold.red(test_counters.failed),
pprint.bold(" test(s) failed,")
))
print(string.format("%d test(s) passed,", test_counters.passed))
print(string.format("%d test(s) skipped,", test_counters.skipped))
print(string.format("%d test(s) dnr.", test_counters.dnr))
print(string.rep("-",80))
end
@ -591,15 +632,16 @@ local run_tests = function()
local current_title = {}
for _, test in ipairs(tests) do
if not test.func then
nb_tests = nb_tests + 1
table.insert(current_title, test)
elseif test.players and next(test.players) then
nb_tests = nb_tests + 1
for _, t in ipairs(current_title) do
table.insert(players_tests, t)
end
current_title = {}
table.insert(players_tests, test)
else
nb_tests = nb_tests + 1
for _, t in ipairs(current_title) do
table.insert(simple_tests, t)
end
@ -611,8 +653,8 @@ local run_tests = function()
print("Running " .. nb_tests .. " tests for:")
for _,context in ipairs(tests_context) do
print(" - "..context.mod .." - "..context.version_string)
for _,context in pairs(tests_context) do
print(" - "..context.mod .." - "..(context.version_string or ""))
end
print("on " .. v.project .. " " .. (v.hash or v.string))
end
@ -659,10 +701,10 @@ local run_tests = function()
end
if tests_state == TESTS_STATE_ENUM.DONE then
display_tests_summary()
if minetest.settings:get_bool("test_harness_stop_server", true) then
request_shutdown()
end
return
end
-- list of needed players

6
init.lua
View File

@ -8,9 +8,5 @@ test_harness.version_string = string.format("%d.%d", ver.major, ver.minor)
if minetest.settings:get_bool("test_harness_run_tests", false) then
dofile(test_harness.modpath .. "/base.lua")
end
if minetest.settings:get_bool("test_harness_run_internal_tests", false) then
test_harness.set_context("test_harness", test_harness.version_string)
dofile(test_harness.modpath .. "/test/init.lua")
dofile(test_harness.modpath .. "/test/init.lua")
end

2
settingtypes.txt
View File

@ -1,5 +1,5 @@
test_harness_run_tests (Run registered tests) bool false
test_harness_run_internal_tests (Run registered tests) bool false
test_harness_mods (Comma separated list of mod to run the tests for. Empty means all mods) string ""
test_harness_failfast (Stops at the first error) bool false
test_harness_stop_server (Stop the server after the end of tests) bool true
test_harness_test_players_password (Password for the test players) string test

2
test/init.lua
View File

@ -1,4 +1,4 @@
local register_test = test_harness.get_test_registrator("test_harness")
local register_test = test_harness.get_test_registrator("test_harness", test_harness.version_string)
local get_node = minetest.get_node
local set_node = minetest.set_node
local air = "air"

2
test/players.lua
View File

@ -1,4 +1,4 @@
local register_test = test_harness.get_test_registrator("test_harness")
local register_test = test_harness.get_test_registrator("test_harness", test_harness.version_string)
local vec = vector.new
register_test("Tests utilities for players")