Add 'serialize_lib/' from commit 'a6e8b8b4353863ad563a4d5187f40fea702ea2de'

git-subtree-dir: serialize_lib
git-subtree-mainline: b82e10051d69730b7459cceae4e4e8719b4445d0
git-subtree-split: a6e8b8b4353863ad563a4d5187f40fea702ea2de

serialize_lib/atomic.lua

@@ -0,0 +1,217 @@
+-- atomic.lua
+-- Utilities for transaction-like handling of serialized state files
+-- Also for multiple files that must be synchronous, as advtrains currently requires.
+
+
+-- Managing files and backups
+-- ==========================
+
+--[[
+The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
+the write, so we have incomplete data. So, the following scheme is applied:
+Unix:
+1. writes to <filename>.new
+2. moves <filename>.new to <filename>, clobbering previous file
+Windows:
+1. writes to <filename>.new
+2. delete <filename>
+3. moves <filename>.new to <filename>
+
+We count a new version of the state as "committed" after stage 2.
+
+During loading, we apply the following order of precedence:
+1. <filename>
+2. <filename>.new (windows only, in case we were interrupted just before 3. when saving)
+
+
+All of these functions return either true on success or nil, error on error.
+]]--
+
+local ser = serialize_lib.serialize
+
+local windows_mode = false
+
+-- == local functions ==
+
+local function save_atomic_move_file(filename)
+	--2. if windows mode, delete main file
+	if windows_mode then
+		local delsucc, err = os.remove(filename)
+		if not delsucc then
+			serialize_lib.log_warn("Unable to delete old savefile '"..filename.."':")
+			serialize_lib.log_warn(err)
+			serialize_lib.log_info("Trying to replace the save file anyway now...")
+		end
+	end
+	
+	--3. move file
+	local mvsucc, err = os.rename(filename..".new", filename)
+	if not mvsucc then
+		if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then
+			serialize_lib.log_error("Unable to replace save file '"..filename.."':")
+			serialize_lib.log_error(err)
+			return nil, err
+		else
+			-- enable windows mode and try again
+			serialize_lib.log_info("Unable to replace save file '"..filename.."' by direct renaming:")
+			serialize_lib.log_info(err)
+			serialize_lib.log_info("Enabling Windows mode for atomic saving...")
+			windows_mode = true
+			return save_atomic_move_file(filename)
+		end
+	end
+	
+	return true
+end
+
+local function open_file_and_save_callback(callback, filename)
+	local file, err = io.open(filename, "w")
+	if not file then
+		error("Failed opening file '"..filename.."' for write:\n"..err)
+	end
+	
+	callback(file)
+	return true
+end
+
+local function open_file_and_load_callback(filename, callback)
+	local file, err = io.open(filename, "r")
+	if not file then
+		error("Failed opening file '"..filename.."' for read:\n"..err)
+	end
+	
+	return callback(file)
+end
+
+-- == public functions ==
+
+-- Load a saved state (according to comment above)
+-- if 'callback' is nil: reads serialized table.
+-- returns the read table, or nil,err on error
+-- if 'callback' is a function (signature func(file_handle) ):
+-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it.
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+function serialize_lib.load_atomic(filename, callback)
+	
+	local cbfunc = callback or ser.read_from_fd
+	
+	-- try <filename>
+	local file, ret = io.open(filename, "r")
+	if file then
+		-- read the file using the callback
+		local success
+		success, ret = pcall(cbfunc, file)
+		if success then
+			return ret
+		end
+	end
+	
+	if minetest.settings:get_bool("serialize_lib_strict_loading") then
+		serialize_lib.save_lock = true
+		error("Loading data from file '"..filename.."' failed:\n"
+				..ret.."\nDisable Strict Loading to ignore.")
+	end
+		
+	serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:")
+	serialize_lib.log_warn(ret)
+	
+	-- try <filename>.new
+	file, ret = io.open(filename..".new", "r")
+	if file then
+		-- read the file using the callback
+		local success
+		success, ret = pcall(cbfunc, file)
+		if success then
+			return ret
+		end
+	end
+	
+	serialize_lib.log_error("Unable to load data from '"..filename..".new':")
+	serialize_lib.log_error(ret)
+	serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.")
+	
+	return nil, ret
+end
+
+-- Save a file atomically (as described above)
+-- 'data' is the data to be saved (when a callback is used, this can be nil)
+-- if 'callback' is nil:
+-- data must be a table, and is serialized into the file
+-- if 'callback' is a function (signature func(data, file_handle) ):
+-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic().
+-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
+-- The callback's first return value is returned by load_atomic
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic(data, filename, callback, config)
+	if serialize_lib.save_lock then
+		serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!")
+		return nil
+	end
+	
+	local cbfunc = callback or ser.write_to_fd
+
+	local file, ret = io.open(filename..".new", "w")
+	if file then
+		-- save the file using the callback
+		local success
+		success, ret = pcall(cbfunc, data, file)
+		if success then
+			return save_atomic_move_file(filename)
+		end
+	end
+	serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+	serialize_lib.log_error(ret)
+	return nil, ret
+end
+
+
+-- Saves multiple files synchronously. First writes all data to all <filename>.new files,
+-- then moves all files in quick succession to avoid inconsistent backups.
+-- parts_table is a table where the keys are used as part of the filename and the values
+-- are the respective data written to it.
+-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out.
+-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does.
+-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar')
+-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument.
+-- Important: the callback must close the file in all cases!
+function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config)
+	if serialize_lib.save_lock then
+		serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!")
+		return nil
+	end
+	
+	for subfile, data in pairs(parts_table) do
+		local filename = filename_prefix..subfile
+		local cbfunc = ser.write_to_fd
+		if callbacks_table and callbacks_table[subfile] then
+			cbfunc = callbacks_table[subfile]
+		end
+		
+		local success = false
+		local file, ret = io.open(filename..".new", "w")
+		if file then
+			-- save the file using the callback
+			success, ret = pcall(cbfunc, data, file, config)
+		end
+		
+		if not success then
+			serialize_lib.log_error("Unable to save data to '"..filename..".new':")
+			serialize_lib.log_error(ret)
+			return nil, ret
+		end
+	end
+	
+	local first_error
+	for file, _ in pairs(parts_table) do
+		local filename = filename_prefix..file
+		local succ, err = save_atomic_move_file(filename)
+		if not succ and not first_error then
+			first_error = err
+		end
+	end
+	
+	return not first_error, first_error -- either true,nil or nil,error
+end
+
+

serialize_lib/init.lua

@@ -0,0 +1,74 @@
+-- serialize_lib
+--[[
+	Copyright (C) 2020  Moritz Blei (orwell96) and contributors
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+]]--
+
+serialize_lib = {}
+
+--[[ Configuration table
+Whenever asked for a "config", the following table structure is expected:
+config = {
+	skip_empty_tables = false	-- if true, does not store empty tables
+								-- On next read, keys that mapped to empty tables resolve to nil
+								-- Used by: write_table_to_file
+}
+Not all functions use all of the parameters, so you can simplify your config sometimes
+]]
+
+-- log utils
+-- =========
+
+
+function serialize_lib.log_error(text)
+	minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_warn(text)
+	minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_info(text)
+	minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
+end
+function serialize_lib.log_debug(text)
+	minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>"))
+end
+
+-- basic serialization/deserialization
+-- ===================================
+
+local mp = minetest.get_modpath(minetest.get_current_modname())
+serialize_lib.serialize = dofile(mp.."/serialize.lua")
+dofile(mp.."/atomic.lua")
+
+local ser = serialize_lib.serialize
+
+-- Opens the passed filename, and returns deserialized table
+-- When an error occurs, logs an error and returns false
+function serialize_lib.read_table_from_file(filename)
+	local succ, ret = pcall(ser.read_from_file, filename)
+	if not succ then
+		serialize_lib.log_error(ret)
+	end
+	return ret
+end
+
+-- Writes table into file
+-- When an error occurs, logs an error and returns false
+function serialize_lib.write_table_to_file(root_table, filename)
+	local succ, ret = pcall(ser.write_to_file, root_table, filename)
+	if not succ then
+		serialize_lib.log_error(ret)
+	end
+	return ret
+end
+
+

serialize_lib/mod.conf

@@ -0,0 +1,4 @@
+name=serialize_lib
+title=Lua Serialization Library
+description=A library to efficiently write and read Lua tables to/from a file, with support for atomic operations
+author=orwell96

serialize_lib/readme.md

@@ -0,0 +1,7 @@
+# serialize_lib
+A Minetest mod library for safely storing large amounts of data in on-disk files.
+Created out of the need to have a robust data store for advtrains.
+
+The main purpose is to load and store large Lua table structures into files, without loading everything in memory and exhausting the function constant limit of LuaJIT.
+
+Also contains various utilities to handle files on disk in a safe manner, retain multiple versions of the same file a.s.o.

serialize_lib/serialize.lua

@@ -0,0 +1,269 @@
+-- serialize.lua
+-- Lua-conformant library file that has no minetest dependencies
+-- Contains the serialization and deserialization routines
+
+--[[
+
+Structure of entry:
+[keytype][key]:[valuetype][val]
+Types:
+	B - bool
+		-> 0=false, 1=true
+    S - string
+		-> see below
+    N - number
+		-> thing compatible with tonumber()
+Table:
+[keytype][key]:T
+... content is nested in table until the matching
+E
+
+example:
+LUA_SER v=1		{
+Skey:Svalue			key = "value",
+N1:Seins			[1] = "eins",
+B1:T				[true] = {
+Sa:Sb					a = "b",
+Sc:B0					c = false,
+E					}
+E				}
+
+String representations:
+In strings the following characters are escaped by &
+'&' -> '&&'
+(line break) -> '&n'
+(CR) -> '&r'			--> required?
+':' -> '&:'
+All other characters are unchanged as they bear no special meaning.
+]]
+
+local write_table, literal_to_string, escape_chars, table_is_empty
+
+function table_is_empty(t)
+	for _,_ in pairs(t) do
+		return false
+	end
+	return true
+end
+
+function write_table(t, file, config)
+	local ks, vs, writeit, istable
+	for key, value in pairs(t) do
+		ks = value_to_string(key, false)
+		writeit = true
+		istable = type(value)=="table"
+		
+		if istable then
+			vs = "T"
+			if config and config.skip_empty_tables then
+				writeit = not table_is_empty(value)
+			end
+		else
+			vs = value_to_string(value, true)
+		end
+		
+		if writeit then
+			file:write(ks..":"..vs.."\n")
+			
+			if istable then
+				write_table(value, file, config)
+				file:write("E\n")
+			end
+		end
+	end
+end
+
+function value_to_string(t)
+	if type(t)=="table" then
+		file:close()
+		error("Can not serialize a table in the key position!")
+	elseif type(t)=="boolean" then
+		if t then
+			return "B1"
+		else
+			return "B0"
+		end
+	elseif type(t)=="number" then
+		return "N"..t
+	elseif type(t)=="string" then
+		return "S"..escape_chars(t)
+	else
+		--error("Can not serialize '"..type(t).."' type!")
+		return "S<function>"
+	end
+	return str
+end
+
+function escape_chars(str)
+	local rstr = string.gsub(str, "&", "&&")
+	rstr = string.gsub(rstr, ":", "&:")
+	rstr = string.gsub(rstr, "\n", "&n")
+	return rstr
+end
+
+------
+
+local read_table, string_to_value, unescape_chars
+
+function read_table(t, file)
+	local line, ks, vs, kv, vv, vt
+	while true do
+		line = file:read("*l")
+		if not line then
+			file:close()
+			error("Unexpected EOF or read error!")
+		end
+		
+		if line=="E" then
+			-- done with this table
+			return
+		end
+		ks, vs = string.match(line, "^(.*[^&]):(.+)$")
+		if not ks or not vs then
+			file:close()
+			error("Unable to parse line: '"..line.."'!")
+		end
+		kv = string_to_value(ks)
+		vv, vt = string_to_value(vs, true)
+		if vt then
+			read_table(vv, file)
+		end
+		-- put read value in table
+		t[kv] = vv
+	end
+end
+
+-- returns: value, is_table
+function string_to_value(str, table_allow)
+	local first = string.sub(str, 1,1)
+	local rest = string.sub(str, 2)
+	if first=="T" then
+		if table_allow then
+			return {}, true
+		else
+			file:close()
+			error("Table not allowed in key component!")
+		end
+	elseif first=="N" then
+		local num = tonumber(rest)
+		if num then
+			return num
+		else
+			file:close()
+			error("Unable to parse number: '"..rest.."'!")
+		end
+	elseif first=="B" then
+		if rest=="0" then
+			return false
+		elseif rest=="1" then
+			return true
+		else
+			file:close()
+			error("Unable to parse boolean: '"..rest.."'!")
+		end
+	elseif first=="S" then
+		return unescape_chars(rest)
+	else
+		file:close()
+		error("Unknown literal type '"..first.."' for literal '"..str.."'!")
+	end
+end
+
+function unescape_chars(str) --TODO
+	local rstr = string.gsub(str, "&:", ":")
+	rstr = string.gsub(rstr, "&n", "\n")
+	rstr = string.gsub(rstr, "&&", "&")
+	return rstr
+end
+
+------
+
+--[[
+config = {
+	skip_empty_tables = false	-- if true, does not store empty tables
+								-- On next read, keys that mapped to empty tables resolve to nil
+}
+]]
+
+-- Writes the passed table into the passed file descriptor, and closes the file
+local function write_to_fd(root_table, file, config)
+	file:write("LUA_SER v=1\n")
+	write_table(root_table, file, config)
+	file:write("E\nEND_SER\n")
+	file:close()
+end
+
+-- Reads the file contents from the passed file descriptor and returns the table on success
+-- Throws errors when something is wrong. Closes the file.
+-- config: see above
+local function read_from_fd(file)
+	local first_line = file:read("*l")
+	if first_line ~= "LUA_SER v=1" then
+		file:close()
+		error("Expected header, got '"..first_line.."' instead!")
+	end
+	local t = {}
+	read_table(t, file)
+	local last_line = file:read("*l")
+	file:close()
+	if last_line ~= "END_SER" then
+		error("Missing END_SER, got '"..last_line.."' instead!")
+	end
+	return t
+end
+
+-- Opens the passed filename and serializes root_table into it
+-- config: see above
+function write_to_file(root_table, filename, config)
+	-- try opening the file
+	local file, err = io.open(filename, "w")
+	if not file then
+		error("Failed opening file '"..filename.."' for write:\n"..err)
+	end
+	
+	write_to_fd(root_table, file, config)
+	return true
+end
+
+-- Opens the passed filename, and returns its deserialized contents
+function read_from_file(filename)
+	-- try opening the file
+	local file, err = io.open(filename, "r")
+	if not file then
+		error("Failed opening file '"..filename.."' for read:\n"..err)
+	end
+	
+	return read_from_fd(file)
+end
+
+--[[ simple unit test
+local testtable = {
+	key = "value",
+	[1] = "eins",
+	[true] = {
+		a = "b",
+		c = false,
+	},
+	["es:cape1"] = "foo:bar",
+	["es&ca\npe2"] = "baz&bam\nbim",
+	["es&&ca&\npe3"] = "baz&&bam&\nbim",
+	["es&:cape4"] = "foo\n:bar"
+}
+local config = {}
+--write_to_file(testtable, "test_out", config)
+local t = read_from_file("test_out")
+write_to_file(t, "test_out_2", config)
+local t2 = read_from_file("test_out_2")
+write_to_file(t2, "test_out_3", config)
+
+-- test_out_2 and test_out_3 should be equal
+
+--]]
+
+
+return {
+	read_from_fd = read_from_fd,
+	write_to_fd = write_to_fd,
+	read_from_file = read_from_file,
+	write_to_file = write_to_file,
+}

serialize_lib/settingtypes.txt

@@ -0,0 +1,12 @@
+# Enable strict file loading mode
+# If enabled, if any error occurs during loading of a file using the 'atomic' API,
+# an error is thrown. You probably need to disable this option for initial loading after
+# creating the world.
+serialize_lib_strict_loading (Strict loading) bool false
+
+# Do not automatically switch to "Windows mode" when saving atomically
+# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically
+# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent
+# this behavior and abort saving instead.
+serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false
+

serialize_lib/tests/serialize_spec.lua

@@ -0,0 +1,123 @@
+-- test the serialization function
+
+
+package.path  =  "../?.lua;" .. package.path
+
+
+ser = require("serialize")
+
+
+local mock_file = {}
+_G.mock_file = mock_file
+function mock_file:read(arg)
+	if arg == "*l" then
+		local l = self.lines[self.pointer or 1]
+		self.pointer = (self.pointer or 1) + 1
+		return l
+	end
+end
+
+function mock_file:close()
+	return nil
+end
+
+function mock_file:write(text)
+	self.content = self.content..text
+end
+
+function mock_file:create(lines)
+	local f = {}
+	setmetatable(f, mock_file)
+	f.lines = lines or {}
+	f.write = self.write
+	f.close = self.close
+	f.read = self.read
+	f.content = ""
+	return f
+end
+
+
+local testtable = {
+	key = "value",
+	[1] = "eins",
+	[true] = {
+		a = "b",
+		c = false,
+	},
+	["es:cape1"] = "foo:bar",
+	["es&ca\npe2"] = "baz&bam\nbim",
+	["es&&ca&\npe3"] = "baz&&bam&\nbim",
+	["es&:cape4"] = "foo\n:bar"
+}
+local testser = [[LUA_SER v=1
+B1:T
+Sa:Sb
+Sc:B0
+E
+Skey:Svalue
+Ses&&&&ca&&&npe3:Sbaz&&&&bam&&&nbim
+N1:Seins
+Ses&&&:cape4:Sfoo&n&:bar
+Ses&&ca&npe2:Sbaz&&bam&nbim
+Ses&:cape1:Sfoo&:bar
+E
+END_SER
+]]
+
+local function check_write(tb, conf)
+	f = mock_file:create()
+	ser.write_to_fd(tb, f, conf or {})
+	return f.content
+end
+
+function string:split()
+	local fields = {}
+   self:gsub("[^\n]+", function(c) fields[#fields+1] = c end)
+   return fields
+end
+
+local function check_read(text)
+	f = mock_file:create(text:split())
+	return ser.read_from_fd(f)
+end
+	
+local noskip = [[LUA_SER v=1
+N1:T
+E
+E
+END_SER
+]]
+local skip = [[LUA_SER v=1
+E
+END_SER
+]]
+
+describe("write_to_fd", function()
+				it("serializes a table correctly", function()
+				assert.equals(check_write(testtable), testser)
+				end)
+				it("does not skip empty tables", function()
+						assert.equals(check_write({{}}),noskip)
+				end)
+				it("skips empty tables when needed", function()
+
+						assert.equals(check_write({{}},{skip_empty_tables=true}),skip)
+				end)
+end)
+
+describe("read_from_fd", function ()
+				it("reads a table correctly", function()
+						assert.same(check_read(testser),testtable)
+				end)
+				it("handles some edge cases correctly", function()
+						assert.same(check_read(noskip), {{}})
+						assert.same(check_read(skip), {})
+				end)
+				it("Read back table", function()
+						local tb = {}
+						for k=1,262 do
+							tb[k] =  { "Foo", "bar", k}
+						end
+						assert.same(check_read(check_write(tb)), tb)
+				end)
+end)