Yves-Marie Haussonne 8de4cc3964 feat(tests): enhance test framework for minetest mods
- Enhance instructions in the README for setting up and running tests with the test framework.
- Develop modules for registering and running tests within mods.
- Provide a mechanism to control which mods are tested through `test_harness_mods` setting.
- Improve summary display logic for test results and categorize them (passed, failed, skipped, dnr).
- Refactor scripts for better management of test execution and handling various cases.
2024-10-06 09:19:43 +02:00

745 lines
23 KiB
Lua

local tests = {}
local tests_by_mod = {}
local TESTS_STATE_ENUM = {
NOT_STARTED = "not_started",
STARTED = "started",
STARTED_PLAYERS = "started_players",
DONE = "done"
}
local tests_state = TESTS_STATE_ENUM.NOT_STARTED
local tests_context = {}
local tests_mod_list = {}
local test_mods_str = minetest.settings:get("test_harness_mods") or ""
for str in string.gmatch(test_mods_str, "[^,]+") do
str = str:gsub("%s+", "")
tests_mod_list[str] = true
end
local failed = 0
local set_context = function(mod, version_string)
tests_context[mod] = { mod = mod, version_string = version_string}
end
local register_test = function(mod, name, func, opts)
local modnames = minetest.get_modnames()
local is_mod = false
for i=1,#modnames do
if modnames[i]==mod then
is_mod=true
break
end
end
assert(is_mod)
assert(type(name) == "string")
assert(func == nil or type(func) == "function")
if not opts then
opts = {}
else
opts = test_harness.table_copy(opts)
end
opts.mod = mod
opts.name = name
opts.func = func
table.insert(tests, opts)
local mod_test_list = tests_by_mod[mod] or {}
tests_by_mod[mod] = mod_test_list
table.insert(mod_test_list, opts)
end
test_harness.get_test_registrator = function(mod, version_string)
local modnames = minetest.get_modnames()
local is_mod = false
for i=1,#modnames do
if modnames[i]==mod then
is_mod=true
break
end
end
if not is_mod then error("get_test_registrator given mod "..mod.." is not a mod.") end
if #tests_mod_list == 0 or tests_mod_list[mod] then
set_context(mod, version_string)
return function(name, func, opts)
register_test(mod, name, func, opts)
end
else
return function() end
end
end
---------------------
-- Helpers
---------------------
local vec = vector.new
local vecw = function(axis, n, base)
local ret = vec(base)
ret[axis] = n
return ret
end
local pos2str = minetest.pos_to_string
local get_node = minetest.get_node
local set_node = minetest.set_node
local charset = "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890"
math.randomseed(os.clock())
function test_harness.randomString(length)
local ret = {}
local r
for _ = 1, length do
r = math.random(1, #charset)
table.insert(ret, charset:sub(r, r))
end
return table.concat(ret)
end
function test_harness.table_copy(t)
local t2 = {}
for k, v in pairs(t) do
if type(v) == "table" then
t2[k] = test_harness.table_copy(v)
else
t2[k] = v
end
end
return t2
end
--- Copies and modifies positions `pos1` and `pos2` so that each component of
-- `pos1` is less than or equal to the corresponding component of `pos2`.
-- Returns the new positions.
function test_harness.sort_pos(pos1, pos2)
pos1 = vector.copy(pos1)
pos2 = vector.copy(pos2)
if pos1.x > pos2.x then
pos2.x, pos1.x = pos1.x, pos2.x
end
if pos1.y > pos2.y then
pos2.y, pos1.y = pos1.y, pos2.y
end
if pos1.z > pos2.z then
pos2.z, pos1.z = pos1.z, pos2.z
end
return pos1, pos2
end
--- Determines the volume of the region defined by positions `pos1` and `pos2`.
-- @return The volume.
function test_harness.volume(pos1, pos2)
local pos1, pos2 = test_harness.sort_pos(pos1, pos2)
return (pos2.x - pos1.x + 1) *
(pos2.y - pos1.y + 1) *
(pos2.z - pos1.z + 1)
end
---------------------
-- Nodes
---------------------
local air = "air"
rawset(_G, "testnode1", "")
rawset(_G, "testnode2", "")
rawset(_G, "testnode3", "")
-- Loads nodenames to use for tests
local function init_nodes()
testnode1 = minetest.registered_aliases["mapgen_stone"]
testnode2 = minetest.registered_aliases["mapgen_dirt"]
testnode3 = minetest.registered_aliases["mapgen_cobble"] or minetest.registered_aliases["mapgen_dirt_with_grass"]
assert(testnode1 and testnode2 and testnode3)
end
-- Writes repeating pattern into given area
rawset(_G, "place_pattern", function(pos1, pos2, pattern)
local pos = vec()
local node = { name = "" }
local i = 1
for z = pos1.z, pos2.z do
pos.z = z
for y = pos1.y, pos2.y do
pos.y = y
for x = pos1.x, pos2.x do
pos.x = x
node.name = pattern[i]
set_node(pos, node)
i = i % #pattern + 1
end
end
end
end)
---------------------
-- Area management
---------------------
assert(minetest.get_mapgen_setting("mg_name") == "singlenode")
rawset(_G, "area", {})
do
local areamin, areamax
local off
local c_air = minetest.get_content_id(air)
local vbuffer = {}
-- Assign a new area for use, will emerge and then call ready()
area.assign = function(min, max, ready)
areamin = min
areamax = max
minetest.emerge_area(min, max, function(bpos, action, remaining)
assert(action ~= minetest.EMERGE_ERRORED)
if remaining > 0 then return end
minetest.after(0, function()
area.clear()
ready()
end)
end)
end
-- Reset area contents and state
area.clear = function()
if off and vector.equals(off, vec(0, 0, 0)) then
return
end
local vmanip = minetest.get_voxel_manip(areamin, areamax)
local vpos1, vpos2 = vmanip:get_emerged_area()
local vcount = (vpos2.x - vpos1.x + 1) * (vpos2.y - vpos1.y + 1) * (vpos2.z - vpos1.z + 1)
if #vbuffer ~= vcount then
vbuffer = {}
for i = 1, vcount do
vbuffer[i] = c_air
end
end
vmanip:set_data(vbuffer)
vmanip:write_to_map()
off = vec(0, 0, 0)
end
-- Returns an usable area [pos1, pos2] that does not overlap previous ones
area.get = function(sizex, sizey, sizez)
local size
if sizey == nil and sizez == nil then
size = vec(sizex, sizex, sizex)
else
size = vec(sizex, sizey, sizez)
end
local pos1 = vector.add(areamin, off)
local pos2 = vector.subtract(vector.add(pos1, size), 1)
if pos2.x > areamax.x or pos2.y > areamax.y or pos2.z > areamax.z then
error("Internal failure: out of space")
end
off = vector.add(off, size)
return pos1, pos2
end
-- Returns an axis and count (= n) relative to the last-requested area that is unoccupied
area.dir = function(n)
local pos1 = vector.add(areamin, off)
if pos1.x + n <= areamax.x then
off.x = off.x + n
return "x", n
elseif pos1.x + n <= areamax.y then
off.y = off.y + n
return "y", n
elseif pos1.z + n <= areamax.z then
off.z = off.z + n
return "z", n
end
error("Internal failure: out of space")
end
-- Returns [XYZ] margin (list of pos pairs) of n around last-requested area
-- (may actually be larger but doesn't matter)
area.margin = function(n)
local pos1, pos2 = area.get(n)
return {
{ vec(areamin.x, areamin.y, pos1.z), pos2 }, -- X/Y
{ vec(areamin.x, pos1.y, areamin.z), pos2 }, -- X/Z
{ vec(pos1.x, areamin.y, areamin.z), pos2 }, -- Y/Z
}
end
end
-- Split an existing area into two non-overlapping [pos1, half1], [half2, pos2] parts; returns half1, half2
area.split = function(pos1, pos2)
local axis
if pos2.x - pos1.x >= 1 then
axis = "x"
elseif pos2.y - pos1.y >= 1 then
axis = "y"
elseif pos2.z - pos1.z >= 1 then
axis = "z"
else
error("Internal failure: area too small to split")
end
local hspan = math.floor((pos2[axis] - pos1[axis] + 1) / 2)
local half1 = vecw(axis, pos1[axis] + hspan - 1, pos2)
local half2 = vecw(axis, pos1[axis] + hspan, pos2)
return half1, half2
end
---------------------
-- Checks
---------------------
rawset(_G, "check", {})
-- Check that all nodes in [pos1, pos2] are the node(s) specified
check.filled = function(pos1, pos2, nodes)
if type(nodes) == "string" then
nodes = { nodes }
end
local _, counts = minetest.find_nodes_in_area(pos1, pos2, nodes)
local total = test_harness.volume(pos1, pos2)
local sum = 0
for _, n in pairs(counts) do
sum = sum + n
end
if sum ~= total then
error((total - sum) .. " " .. table.concat(nodes, ",") .. " nodes missing in " ..
pos2str(pos1) .. " -> " .. pos2str(pos2))
end
end
-- Check that none of the nodes in [pos1, pos2] are the node(s) specified
check.not_filled = function(pos1, pos2, nodes)
if type(nodes) == "string" then
nodes = { nodes }
end
local _, counts = minetest.find_nodes_in_area(pos1, pos2, nodes)
for nodename, n in pairs(counts) do
if n ~= 0 then
error(counts[nodename] .. " " .. nodename .. " nodes found in " ..
pos2str(pos1) .. " -> " .. pos2str(pos2))
end
end
end
-- Check that all of the areas are only made of node(s) specified
check.filled2 = function(list, nodes)
for _, pos in ipairs(list) do
check.filled(pos[1], pos[2], nodes)
end
end
-- Check that none of the areas contain the node(s) specified
check.not_filled2 = function(list, nodes)
for _, pos in ipairs(list) do
check.not_filled(pos[1], pos[2], nodes)
end
end
-- Checks presence of a repeating pattern in [pos1, po2] (cf. place_pattern)
check.pattern = function(pos1, pos2, pattern)
local pos = vec()
local i = 1
for z = pos1.z, pos2.z do
pos.z = z
for y = pos1.y, pos2.y do
pos.y = y
for x = pos1.x, pos2.x do
pos.x = x
local node = get_node(pos)
if node.name ~= pattern[i] then
error(pattern[i] .. " not found at " .. pos2str(pos) .. " (i=" .. i .. ")")
end
i = i % #pattern + 1
end
end
end
end
---------------------------------------
--- Colors Management
--- from https://github.com/ldrumm/lua-chroma
--------------------------------------
rawset(_G, "pprint", setmetatable({
escapes = {
clear = "\027[0m",
red = "\027[31m",
green = "\027[32m",
orange = "\027[33m",
navy = "\027[34m",
magenta = "\027[35m",
cyan = "\027[36m",
gray = "\027[90m",
grey = "\027[90m",
light_gray = "\027[37m",
light_grey = "\027[37m",
peach = "\027[91m",
light_green = "\027[92m",
yellow = "\027[93m",
blue = "\027[94m",
pink = "\027[95m",
baby_blue = "\027[96m",
highlight = {
red = "\027[41m",
green = "\027[42m",
orange = "\027[43m",
navy = "\027[44m",
magenta = "\027[45m",
cyan = "\027[46m",
gray = "\027[47m",
grey = "\027[47m",
light_gray = "\027[100m",
light_grey = "\027[100m",
peach = "\027[101m",
light_green = "\027[102m",
yellow = "\027[103m",
blue = "\027[104m",
pink = "\027[105m",
baby_blue = "\027[106m",
},
strikethrough = "\027[9m",
underline = "\027[4m",
bold = "\027[1m",
},
_sequence = '',
_highlight = false,
},
{
__call = function(self, ...)
local arg = {...}
local add_sequence = (not not self._sequence) and #self._sequence > 0
local call_res = (add_sequence and self._sequence or '')
for _,v in ipairs(arg) do
call_res = call_res .. v
end
self._sequence = ''
return call_res .. ((add_sequence and rawget(self, "escapes").clear) or '')
end,
__index = function(self, index)
local esc = self._highlight and rawget(self, 'escapes').highlight[index]
or rawget(self, 'escapes')[index]
self._highlight = index == 'highlight'
if esc ~= nil then
if type(esc) == 'string' then
if index == 'clear' then
self._sequence = ""
else
self._sequence = self._sequence .. esc
end
end
return self
else
return rawget(self, index)
end
end,
}))
--------------------------------------
local request_shutdown = function()
if failed == 0 then
io.close(io.open(minetest.get_worldpath() .. "/tests_ok", "w"))
end
print("Requesting server shutdown")
minetest.request_shutdown()
end
local all_in_table = function(names, names_list)
for _, n in ipairs(names) do
if not names_list[n] then
return false
end
end
return true
end
local print_result_line = function(test)
local s = ":"..test.mod..":"
local rest = s .. test.name
print(string.format("%s %s%s%s",
pprint.light_gray(s),
test.name,
string.rep(" ", 80 - #rest),
test.result.ok and pprint.green("pass") or pprint.red("FAIL")
))
if not test.result.ok and test.result.err then
print(pprint.yellow(" " .. test.result.err))
end
end
local display_tests_summary = function()
local title = "TESTS RUN SUMMARY"
local remaining_width = 72 - #title
local left = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width-1)/2
local right = (remaining_width % 2 == 0) and remaining_width/2 or (remaining_width+1)/2
print(string.rep("-",80))
print("----"..string.rep(" ",72).."----")
print(string.format("----%s%s%s----",
string.rep(" ",left),
pprint.bold.underline.orange(title),
string.rep(" ",right)
))
print("----"..string.rep(" ",72).."----")
print(string.rep("-",80))
print("All tests done, " .. failed .. " tests failed.")
print()
local test_counters = { total=0, passed=0, failed=0, skipped=0, dnr=0}
for mod, tests_list in pairs(tests_by_mod) do
print(pprint.baby_blue(string.format("%#80s", mod)))
for _, test in ipairs(tests_list) do
if test.func == nil then
local s = ":".. test.mod ..":---- " .. test.name
print(pprint.light_gray(":".. test.mod ..":")..pprint.blue("---- " .. test.name .. string.rep("-", 80 - #s)))
elseif test.result ~= nil then
test_counters["total"] = test_counters["total"] + 1
if test.result.ok == nil then
test_counters["skipped"] = test_counters["skipped"] + 1
elseif test.result.ok then
test_counters["passed"] = test_counters["passed"] + 1
else
test_counters["failed"] = test_counters["failed"] + 1
end
print_result_line(test)
else
test_counters["total"] = test_counters["total"] + 1
test_counters["dnr"] = test_counters["dnr"] + 1
local s = ":"..test.mod..":"
local rest = s .. test.name
print(pprint.light_gray(s.." "..test.name..string.rep(" ", 80 - #rest).."dnr"))
end
end
print(pprint.baby_blue(string.rep("-",80)))
end
print(string.rep("-",80))
print(string.format("%s%s%s",
pprint.bold(string.format("%d Tests done, ", test_counters.total)),
(test_counters.failed==0 and pprint.bold.green(test_counters.failed)) or pprint.bold.red(test_counters.failed),
pprint.bold(" test(s) failed,")
))
print(string.format("%d test(s) passed,", test_counters.passed))
print(string.format("%d test(s) skipped,", test_counters.skipped))
print(string.format("%d test(s) dnr.", test_counters.dnr))
print(string.rep("-",80))
end
test_harness.dump = function(o)
if type(o) == 'table' then
local s = '{ '
for k, v in pairs(o) do
if type(k) ~= 'number' then k = '"' .. k .. '"' end
s = s .. '[' .. k .. '] = ' .. test_harness.dump(v) .. ','
end
return s .. '} '
else
return tostring(o)
end
end
test_harness.save_players = function(players)
local players_data = {}
for _, p in ipairs(players) do
local player_obj = nil
local player_name = nil
if type(p) == "string" then
player_obj = minetest.get_player_by_name(p)
player_name = p
else
player_obj = p
player_name = p:get_player_name()
end
players_data[player_name] = {
position = player_obj:get_pos(),
privs = test_harness.table_copy(minetest.get_player_privs(player_name)),
inventory = test_harness.table_copy(player_obj:get_inventory():get_lists())
}
end
return players_data
end
test_harness.restore_players = function(players_data)
for player_name, data in pairs(players_data) do
local player = minetest.get_player_by_name(player_name)
player:set_pos(data.position)
minetest.set_player_privs(player_name, data.privs)
player:get_inventory():set_lists(data.inventory)
end
end
local get_connected_player_names = function()
local connected_player_names = {}
for _, p in ipairs(minetest.get_connected_players()) do
connected_player_names[p:get_player_name()] = true
end
return connected_player_names
end
test_harness.run_player_tests = function(list_player_tests, area)
for _, test in ipairs(list_player_tests) do
local connected_player_names = get_connected_player_names()
if not test.result and
test.func and
test.players and
next(test.players) and
all_in_table(test.players, connected_player_names)
then
local player_data = test_harness.save_players(test.players)
area.clear()
local ok, err = pcall(test.func)
test.result = { ok = ok, err = err }
print_result_line(test)
test_harness.restore_players(player_data)
if not ok then
failed = failed + 1
if minetest.settings:get_bool("test_harness_failfast", false) then
tests_state = TESTS_STATE_ENUM.DONE
break
end
end
end
end
local remaining_tests = {}
if tests_state ~= TESTS_STATE_ENUM.DONE then
for _, test in ipairs(list_player_tests) do
if test.func ~= nil and test.result == nil then
table.insert(remaining_tests,test)
end
end
if #remaining_tests == 0 then
tests_state = TESTS_STATE_ENUM.DONE
end
end
if tests_state == TESTS_STATE_ENUM.DONE then
display_tests_summary()
if minetest.settings:get_bool("test_harness_stop_server", true) then
request_shutdown()
end
else
-- reschedule
minetest.after(1,test_harness.run_player_tests,remaining_tests, area)
end
end
---------------------
-- Main function
---------------------
local run_tests = function()
tests_state = TESTS_STATE_ENUM.STARTED
local simple_tests = {}
local players_tests = {}
do
local nb_tests = 0
local current_title = {}
for _, test in ipairs(tests) do
if not test.func then
table.insert(current_title, test)
elseif test.players and next(test.players) then
nb_tests = nb_tests + 1
for _, t in ipairs(current_title) do
table.insert(players_tests, t)
end
current_title = {}
table.insert(players_tests, test)
else
nb_tests = nb_tests + 1
for _, t in ipairs(current_title) do
table.insert(simple_tests, t)
end
current_title = {}
table.insert(simple_tests, test)
end
end
local v = minetest.get_version()
print("Running " .. nb_tests .. " tests for:")
for _,context in pairs(tests_context) do
print(" - "..context.mod .." - "..(context.version_string or ""))
end
print("on " .. v.project .. " " .. (v.hash or v.string))
end
init_nodes()
-- emerge area from (0,0,0) ~ (56,56,56) and keep it loaded
-- Note: making this area smaller speeds up tests
local wanted = vec(56, 56, 56)
for x = 0, math.floor(wanted.x / 16) do
for y = 0, math.floor(wanted.y / 16) do
for z = 0, math.floor(wanted.z / 16) do
assert(minetest.forceload_block(vec(x * 16, y * 16, z * 16), true, -1))
end
end
end
failed = 0
area.assign(vec(0, 0, 0), wanted, function()
-- run the simple tests
for _, test in ipairs(simple_tests) do
if not test.func then
local s = ":"..test.mod..":---- " .. test.name .. " "
print(s .. string.rep("-", 60 - #s))
test.result = { ok = true, err = "" }
else
area.clear()
local ok, err = pcall(test.func)
test.result = { ok = ok, err = err }
print_result_line(test)
if not ok then
failed = failed + 1
if minetest.settings:get_bool("test_harness_failfast", false) then
tests_state = TESTS_STATE_ENUM.DONE
break
end
end
end
end
print("Server tests done, " .. failed .. " tests failed.")
if next(players_tests) == nil then
tests_state = TESTS_STATE_ENUM.DONE
end
if tests_state == TESTS_STATE_ENUM.DONE then
display_tests_summary()
if minetest.settings:get_bool("test_harness_stop_server", true) then
request_shutdown()
end
return
end
-- list of needed players
local players_table = {}
for _, t in ipairs(players_tests) do
if t.players and next(t.players) then
for _, p in ipairs(t.players) do
players_table[p] = true
end
end
end
local needed_playernames = {}
for k, _ in pairs(players_table) do table.insert(needed_playernames, k) end
for _, player_name in ipairs(needed_playernames) do
print("registering player " .. player_name)
minetest.set_player_password(player_name,
minetest.get_password_hash(player_name,
minetest.settings:get("test_harness_test_players_password") or "test"))
end
-- launch test
test_harness.run_player_tests(players_tests, area)
end)
end
-- for debug purposes
minetest.register_on_joinplayer(function(player)
minetest.set_player_privs(player:get_player_name(),
minetest.string_to_privs("fly,fast,noclip,basic_debug,debug,interact"))
end)
minetest.register_on_punchnode(function(pos, node, puncher)
minetest.chat_send_player(puncher:get_player_name(), pos2str(pos))
end)
minetest.after(0, run_tests)