naive zip saving

.luacheckrc

@@ -14,6 +14,9 @@ read_globals = {
 	"dump", "dump2",
 	"VoxelArea",
 
+	-- mods
+	"mtzip",
+
 	-- testing
 	"mtt"
 }

api.lua

@@ -1,3 +1,15 @@
 
-function mapsync.register(def)
+-- name => backend_def
+local backends = {}
+
+-- register a map backend
+function mapsync.register_backend(name, backend_def)
+    backend_def.name = name
+    -- default to always-on backend if no selector specified
+    backend_def.select = backend_def.select or function() return true end
+    backends[name] = backend_def
+end
+
+function mapsync.get_backends()
+    return backends
 end

encode.lua

@@ -0,0 +1,32 @@
+-- https://gist.github.com/mebens/938502
+local function rshift(x, by)
+	return math.floor(x / 2 ^ by)
+end
+
+-- https://stackoverflow.com/a/32387452
+local function bitand(a, b)
+	local result = 0
+	local bitval = 1
+	while a > 0 and b > 0 do
+	  if a % 2 == 1 and b % 2 == 1 then -- test the rightmost bits
+		  result = result + bitval      -- set the current bit
+	  end
+	  bitval = bitval * 2 -- shift left
+	  a = math.floor(a/2) -- shift right
+	  b = math.floor(b/2)
+	end
+	return result
+end
+
+function mapsync.encode_uint16(int)
+	local a, b = int % 0x100, int / 0x100
+	return string.char(a, b)
+end
+
+function mapsync.encode_uint32(v)
+	local b1 = bitand(v, 0xFF)
+	local b2 = bitand( rshift(v, 8), 0xFF )
+	local b3 = bitand( rshift(v, 16), 0xFF )
+	local b4 = bitand( rshift(v, 24), 0xFF )
+	return string.char(b1, b2, b3, b4)
+end

functions.lua

@@ -0,0 +1,39 @@
+
+-- returns a list of backends available for that position
+function mapsync.select_backends(mapblock_pos)
+    local backends = {}
+    for _, backend_def in pairs(mapsync.get_backends()) do
+        if backend_def.select(mapblock_pos) then
+            table.insert(backends, backend_def)
+        end
+    end
+    return backends
+end
+
+--- calculates the mapblock position from a node position
+-- @param pos the node-position
+-- @return the mapblock position
+function mapsync.get_mapblock(pos)
+	return vector.floor( vector.divide(pos, 16) )
+end
+
+--- returns the chunk position from a node position
+-- @param pos the node-position
+-- @return the chunk position
+function mapsync.get_chunkpos(pos)
+	local mapblock_pos = mapsync.get_mapblock(pos)
+	local aligned_mapblock_pos = vector.add(mapblock_pos, 2)
+	return vector.floor( vector.divide(aligned_mapblock_pos, 5) )
+end
+
+function mapsync.get_mapblock_bounds_from_chunk(chunk_pos)
+	local min = vector.subtract( vector.multiply(chunk_pos, 5), 2)
+	local max = vector.add(min, 4)
+	return min, max
+end
+
+function mapsync.get_mapblock_bounds_from_mapblock(mapblock)
+	local min = vector.multiply(mapblock, 16)
+	local max = vector.add(min, 15)
+	return min, max
+end

init.lua

@@ -18,6 +18,9 @@ end
 
 -- pass on global env (secure/insecure)
 loadfile(MP.."/functions.lua")(global_env)
+loadfile(MP.."/serialize.lua")(global_env)
+dofile(MP.."/encode.lua")
+dofile(MP.."/serialize_mapblock.lua")
 dofile(MP.."/api.lua")
 
 if minetest.get_modpath("mtt") and mtt.enabled then

mod.conf

@@ -1,2 +1,3 @@
 name = mapsync
+depends = mtzip
 optional_depends = worldedit, screwdriver2, mtt

mtt.lua

@@ -3,8 +3,11 @@ mtt.register("register and export", function(callback)
     local pos1 = { x=0, y=0, z=0 }
     local pos2 = { x=20, y=20, z=20 }
 
-    mapsync.register({
+    mapsync.register_backend("worldpath", {
-        path = minetest.get_worldpath() .. "/mymap"
+        path = minetest.get_worldpath() .. "/mymap",
+        select = function()
+            return true
+        end
     })
 
     minetest.emerge_area(pos1, pos2, function(_, _, calls_remaining)
@@ -13,3 +16,18 @@ mtt.register("register and export", function(callback)
         end
     end)
 end)
+
+
+mtt.register("serlize_chunk", function(callback)
+    local pos1 = { x=0, y=0, z=0 }
+    local pos2 = { x=20, y=20, z=20 }
+
+    minetest.emerge_area(pos1, pos2, function(_, _, calls_remaining)
+        if calls_remaining == 0 then
+            local success, err_msg = mapsync.serialize_chunk({x=0, y=0, z=0}, minetest.get_worldpath() .. "/chunk.zip")
+            assert(success)
+            assert(not err_msg)
+            callback()
+        end
+    end)
+end)

serialize.lua

@@ -0,0 +1,29 @@
+local global_env = ...
+
+function mapsync.serialize_chunk(chunk_pos, filename)
+    local f = global_env.io.open(filename, "w")
+    local zip = mtzip.zip(f)
+
+    local min, max = mapsync.get_mapblock_bounds_from_chunk(chunk_pos)
+    for x=min.x,max.x do
+        for y=min.y,max.y do
+            for z=min.z,max.z do
+                local mapblock_pos = {x=x, y=y, z=z}
+                local mapblock_data = mapsync.serialize_mapblock(mapblock_pos)
+                if not mapblock_data.empty then
+                    local prefix = "mapblock_" .. minetest.pos_to_string(mapblock_pos)
+                    zip:add(prefix .. "_node_mapping.json", minetest.write_json(mapblock_data.node_mapping))
+                    zip:add(prefix .. "_mapdata.bin", mapblock_data.mapdata)
+                    if mapblock_data.has_metadata then
+                        zip:add(prefix .. "_metadata.json", minetest.write_json(mapblock_data.metadata))
+                    end
+                end
+            end
+        end
+    end
+
+    zip:close()
+    f:close()
+
+    return true
+end

serialize_mapblock.lua

@@ -0,0 +1,155 @@
+
+-- collect nodes with on_timer attributes
+local node_names_with_timer = {}
+minetest.register_on_mods_loaded(function()
+	for _,node in pairs(minetest.registered_nodes) do
+		if node.on_timer then
+			table.insert(node_names_with_timer, node.name)
+		end
+	end
+	minetest.log("action", "[mapsync] collected " .. #node_names_with_timer .. " items with node timers")
+end)
+
+local air_content_id = minetest.get_content_id("air")
+local ignore_content_id = minetest.get_content_id("ignore")
+
+-- map of ignored node_ids (node_id => true)
+local ignore_node_ids = {
+	[ignore_content_id] = true
+}
+
+-- search for other ignored node_ids
+minetest.register_on_mods_loaded(function()
+	for name, node_def in pairs(minetest.registered_nodes) do
+		if node_def.groups and node_def.groups.mapsync_ignore then
+			local node_id = minetest.get_content_id(name)
+			ignore_node_ids[node_id] = true
+		end
+	end
+end)
+
+
+-- local vars for faster access
+local char, encode_uint16, insert = string.char, mapsync.encode_uint16, table.insert
+
+--- Serializes the mapblock at the given position
+-- @param mapblock_pos the mapblock-position
+-- @return @{mapblock_data}
+function mapsync.serialize_mapblock(mapblock_pos)
+	local pos1, pos2 = mapsync.get_mapblock_bounds_from_mapblock(mapblock_pos)
+
+	assert((pos2.x - pos1.x) == 15)
+	assert((pos2.y - pos1.y) == 15)
+	assert((pos2.z - pos1.z) == 15)
+
+	local manip = minetest.get_voxel_manip()
+	local e1, e2 = manip:read_from_map(pos1, pos2)
+	local area = VoxelArea:new({MinEdge=e1, MaxEdge=e2})
+
+	local node_data = manip:get_data()
+	local param1 = manip:get_light_data()
+	local param2 = manip:get_param2_data()
+
+	assert(#node_data == 4096)
+	assert(#param1 == 4096)
+	assert(#param2 == 4096)
+
+	-- prepare data structure
+	local data = {
+		mapdata = {},
+		metadata = {},
+		-- name -> id
+		node_mapping = {},
+		has_metadata = false,
+		empty = true,
+		pos = mapblock_pos
+	}
+
+	local mapdata = {}
+
+	-- id -> nodename
+	local rev_node_mapping = {}
+
+	local j = 1
+
+	-- loop over all blocks and fill cid,param1 and param2
+	for z=pos1.z,pos2.z do
+	for x=pos1.x,pos2.x do
+	for y=pos1.y,pos2.y do
+		local i = area:index(x,y,z)
+
+		local node_id = node_data[i]
+		if ignore_node_ids[node_id] then
+			-- replace ignored blocks with air
+			node_id = air_content_id
+		end
+
+		if node_id ~= air_content_id then
+			data.empty = false
+		end
+
+		-- map node_id
+		if not rev_node_mapping[node_id] then
+			local nodename = minetest.get_name_from_content_id(node_id)
+			rev_node_mapping[node_id] = nodename
+			data.node_mapping[nodename] = node_id
+		end
+
+		mapdata[j] = encode_uint16(node_id)
+		mapdata[j+(4096*2)] = param1[i]
+		mapdata[j+(4096*3)] = param2[i]
+	end
+	end
+	end
+
+	data.mapdata = table.concat(mapdata)
+
+	-- serialize metadata
+	local pos_with_meta = minetest.find_nodes_with_meta(pos1, pos2)
+	for _, meta_pos in ipairs(pos_with_meta) do
+		local relative_pos = vector.subtract(meta_pos, pos1)
+		local meta = minetest.get_meta(meta_pos):to_table()
+
+		-- Convert metadata item stacks to item strings
+		for _, invlist in pairs(meta.inventory) do
+			for index = 1, #invlist do
+				local itemstack = invlist[index]
+				if itemstack.to_string then
+					invlist[index] = itemstack:to_string()
+					data.has_metadata = true
+				end
+			end
+		end
+
+		-- dirty workaround for https://github.com/minetest/minetest/issues/8943
+		if next(meta) and (next(meta.fields) or next(meta.inventory)) then
+			data.has_metadata = true
+			data.metadata.meta = data.metadata.meta or {}
+			data.metadata.meta[minetest.pos_to_string(relative_pos)] = meta
+		end
+
+	end
+
+	-- serialize node timers
+	if #node_names_with_timer > 0 then
+		data.metadata.timers = {}
+		local list = minetest.find_nodes_in_area(pos1, pos2, node_names_with_timer)
+		for _, timer_pos in pairs(list) do
+			local timer = minetest.get_node_timer(timer_pos)
+			local relative_pos = vector.subtract(timer_pos, pos1)
+			if timer:is_started() then
+				data.has_metadata = true
+				local timeout = timer:get_timeout()
+				local elapsed = timer:get_elapsed()
+				data.metadata.timers[minetest.pos_to_string(relative_pos)] = {
+					timeout = timeout,
+					-- round down elapsed timer
+					elapsed = math.min(math.floor(elapsed/10)*10, timeout)
+				}
+			end
+		end
+
+	end
+
+	return data
+end