Calinou
/
minetest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Tweak core.serialize 

This adds proper support for nested tables and improves performance a bit.
master
ShadowNinja 2014-08-03 12:20:35 -04:00
parent 9d69436052
commit 6afdb22ba7
1 changed files with 156 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

326
builtin/common/serialize.lua
View File

@ -1,223 +1,209 @@
-- Minetest: builtin/serialize.lua --- Lua module to serialize values as Lua code.
-- From: https://github.com/fab13n/metalua/blob/no-dll/src/lib/serialize.lua
-- https://github.com/fab13n/metalua/blob/no-dll/src/lib/serialize.lua
-- Copyright (c) 2006-2997 Fabien Fleutot <metalua@gmail.com>
-- License: MIT -- License: MIT
-------------------------------------------------------------------------------- -- @copyright 2006-2997 Fabien Fleutot <metalua@gmail.com>
-- Serialize an object into a source code string. This string, when passed as -- @author Fabien Fleutot <metalua@gmail.com>
-- an argument to deserialize(), returns an object structurally identical -- @author ShadowNinja <shadowninja@minetest.net>
-- to the original one. The following are currently supported:
-- * strings, numbers, booleans, nil
-- * tables thereof. Tables can have shared part, but can't be recursive yet.
-- Caveat: metatables and environments aren't saved.
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
local no_identity = { number=1, boolean=1, string=1, ['nil']=1 } --- Serialize an object into a source code string. This string, when passed as
-- an argument to deserialize(), returns an object structurally identical to
-- the original one. The following are currently supported:
-- * Booleans, numbers, strings, and nil.
-- * Functions; uses interpreter-dependent (and sometimes platform-dependent) bytecode!
-- * Tables; they can cantain multiple references and can be recursive, but metatables aren't saved.
-- This works in two phases:
-- 1. Recursively find and record multiple references and recursion.
-- 2. Recursively dump the value into a string.
-- @param x Value to serialize (nil is allowed).
-- @return load()able string containing the value.
function core.serialize(x) function core.serialize(x)
local local_index = 1 -- Top index of the "_" local table in the dump
-- table->nil/1/2 set of tables seen.
-- nil = not seen, 1 = seen once, 2 = seen multiple times.
local seen = {}
local gensym_max = 0 -- index of the gensym() symbol generator -- nest_points are places where a table appears within itself, directly
local seen_once = { } -- element->true set of elements seen exactly once in the table -- or not. For instance, all of these chunks create nest points in
local multiple = { } -- element->varname set of elements seen more than once -- table x: "x = {}; x[x] = 1", "x = {}; x[1] = x",
local nested = { } -- transient, set of elements currently being traversed -- "x = {}; x[1] = {y = {x}}".
local nest_points = { } -- To handle those, two tables are used by mark_nest_point:
local nest_patches = { } -- * nested - Transient set of tables being currently traversed.
-- Used for detecting nested tables.
local function gensym() -- * nest_points - parent->{key=value, ...} table cantaining the nested
gensym_max = gensym_max + 1 ; return gensym_max -- keys and values in the parent. They're all dumped after all the
end -- other table operations have been performed.
-----------------------------------------------------------------------------
-- nest_points are places where a table appears within itself, directly or not.
-- for instance, all of these chunks create nest points in table x:
-- "x = { }; x[x] = 1", "x = { }; x[1] = x", "x = { }; x[1] = { y = { x } }".
-- To handle those, two tables are created by mark_nest_point:
-- * nest_points [parent] associates all keys and values in table parent which
-- create a nest_point with boolean `true'
-- * nest_patches contain a list of { parent, key, value } tuples creating
-- a nest point. They're all dumped after all the other table operations
-- have been performed.
-- --
-- mark_nest_point (p, k, v) fills tables nest_points and nest_patches with -- mark_nest_point(p, k, v) fills nest_points with information required
-- informations required to remember that key/value (k,v) create a nest point -- to remember that key/value (k, v) creates a nest point in table
-- in table parent. It also marks `parent' as occuring multiple times, since -- parent. It also marks "parent" and the nested item(s) as occuring
-- several references to it will be required in order to patch the nest -- multiple times, since several references to it will be required in
-- points. -- order to patch the nest points.
----------------------------------------------------------------------------- local nest_points = {}
local function mark_nest_point (parent, k, v) local nested = {}
local function mark_nest_point(parent, k, v)
local nk, nv = nested[k], nested[v] local nk, nv = nested[k], nested[v]
assert (not nk or seen_once[k] or multiple[k]) local np = nest_points[parent]
assert (not nv or seen_once[v] or multiple[v]) if not np then
local mode = (nk and nv and "kv") or (nk and "k") or ("v") np = {}
local parent_np = nest_points [parent] nest_points[parent] = np
local pair = { k, v } end
if not parent_np then parent_np = { }; nest_points [parent] = parent_np end np[k] = v
parent_np [k], parent_np [v] = nk, nv seen[parent] = 2
table.insert (nest_patches, { parent, k, v }) if nk then seen[k] = 2 end
seen_once [parent], multiple [parent] = nil, true if nv then seen[v] = 2 end
end end
----------------------------------------------------------------------------- -- First phase, list the tables and functions which appear more than
-- First pass, list the tables and functions which appear more than once in x -- once in x.
----------------------------------------------------------------------------- local function mark_multiple_occurences(x)
local function mark_multiple_occurences (x) local tp = type(x)
if no_identity [type(x)] then return end if tp ~= "table" and tp ~= "function" then
if seen_once [x] then seen_once [x], multiple [x] = nil, true -- No identity (comparison is done by value, not by instance)
elseif multiple [x] then -- pass return
else seen_once [x] = true end end
if seen[x] == 1 then
if type (x) == 'table' then seen[x] = 2
nested [x] = true elseif seen[x] ~= 2 then
for k, v in pairs (x) do seen[x] = 1
if nested[k] or nested[v] then mark_nest_point (x, k, v) else end
mark_multiple_occurences (k)
mark_multiple_occurences (v) if tp == "table" then
nested[x] = true
for k, v in pairs(x) do
if nested[k] or nested[v] then
mark_nest_point(x, k, v)
else
mark_multiple_occurences(k)
mark_multiple_occurences(v)
end end
end end
nested [x] = nil nested[x] = nil
end end
end end
local dumped = { } -- multiply occuring values already dumped in localdefs local dumped = {} -- object->varname set
local localdefs = { } -- already dumped local definitions as source code lines local local_defs = {} -- Dumped local definitions as source code lines
-- mutually recursive functions: -- Mutually recursive local functions:
local dump_val, dump_or_ref_val local dump_val, dump_or_ref_val
-------------------------------------------------------------------- -- If x occurs multiple times, dump the local variable rather than
-- if x occurs multiple times, dump the local var rather than the -- the value. If it's the first time it's dumped, also dump the
-- value. If it's the first time it's dumped, also dump the content -- content in local_defs.
-- in localdefs. function dump_or_ref_val(x)
-------------------------------------------------------------------- if seen[x] ~= 2 then
function dump_or_ref_val (x) return dump_val(x)
if nested[x] then return 'false' end -- placeholder for recursive reference end
if not multiple[x] then return dump_val (x) end local var = dumped[x]
local var = dumped [x] if var then -- Already referenced
if var then return "_[" .. var .. "]" end -- already referenced return var
local val = dump_val(x) -- first occurence, create and register reference end
var = gensym() -- First occurence, create and register reference
table.insert(localdefs, "_["..var.."]="..val) local val = dump_val(x)
dumped [x] = var local i = local_index
return "_[" .. var .. "]" local_index = local_index + 1
var = "_["..i.."]"
table.insert(local_defs, var.." = "..val)
dumped[x] = var
return var
end end
----------------------------------------------------------------------------- -- Second phase. Dump the object; subparts occuring multiple times
-- Second pass, dump the object; subparts occuring multiple times are dumped -- are dumped in local variables which can be referenced multiple
-- in local variables which can be referenced multiple times; -- times. Care is taken to dump local vars in a sensible order.
-- care is taken to dump locla vars in asensible order.
-----------------------------------------------------------------------------
function dump_val(x) function dump_val(x)
local t = type(x) local tp = type(x)
if x==nil then return 'nil' if x == nil then return "nil"
elseif t=="number" then return tostring(x) elseif tp == "number" then return tostring(x)
elseif t=="string" then return string.format("%q", x) elseif tp == "string" then return string.format("%q", x)
elseif t=="boolean" then return x and "true" or "false" elseif tp == "boolean" then return x and "true" or "false"
elseif t=="function" then elseif tp == "function" then
return string.format("loadstring(%q)", string.dump(x)) return string.format("loadstring(%q)", string.dump(x))
elseif t=="table" then elseif tp == "table" then
local acc = { } local vals = {}
local idx_dumped = { } local idx_dumped = {}
local np = nest_points [x] local np = nest_points[x]
for i, v in ipairs(x) do for i, v in ipairs(x) do
if np and np[v] then if not np or not np[i] then
table.insert (acc, 'false') -- placeholder table.insert(vals, dump_or_ref_val(v))
else
table.insert (acc, dump_or_ref_val(v))
end end
idx_dumped[i] = true idx_dumped[i] = true
end end
for k, v in pairs(x) do for k, v in pairs(x) do
if np and (np[k] or np[v]) then if (not np or not np[k]) and
--check_multiple(k); check_multiple(v) -- force dumps in localdefs not idx_dumped[k] then
elseif not idx_dumped[k] then table.insert(vals,
table.insert (acc, "[" .. dump_or_ref_val(k) .. "] = " .. dump_or_ref_val(v)) "["..dump_or_ref_val(k).."] = "
..dump_or_ref_val(v))
end end
end end
return "{ "..table.concat(acc,", ").." }" return "{"..table.concat(vals, ", ").."}"
else else
error ("Can't serialize data of type "..t) error("Can't serialize data of type "..t)
end
end
local function dump_nest_patches()
for _, entry in ipairs(nest_patches) do
local p, k, v = unpack (entry)
assert (multiple[p])
local set = dump_or_ref_val (p) .. "[" .. dump_or_ref_val (k) .. "] = " ..
dump_or_ref_val (v) .. " -- rec "
table.insert (localdefs, set)
end end
end end
mark_multiple_occurences (x) local function dump_nest_points()
local toplevel = dump_or_ref_val (x) for parent, vals in pairs(nest_points) do
dump_nest_patches() for k, v in pairs(vals) do
table.insert(local_defs, dump_or_ref_val(parent)
.."["..dump_or_ref_val(k).."] = "
..dump_or_ref_val(v))
end
end
end
if next (localdefs) then mark_multiple_occurences(x)
return "local _={ }\n" .. local top_level = dump_or_ref_val(x)
table.concat (localdefs, "\n") .. dump_nest_points()
"\nreturn " .. toplevel
if next(local_defs) then
return "local _ = {}\n"
..table.concat(local_defs, "\n")
.."\nreturn "..top_level
else else
return "return " .. toplevel return "return "..top_level
end end
end end
-- Deserialization. -- Deserialization
-- http://stackoverflow.com/questions/5958818/loading-serialized-data-into-a-table
--
local env = { local env = {
loadstring = loadstring, loadstring = loadstring,
} }
local function noop() end
local safe_env = { local safe_env = {
loadstring = noop, loadstring = function() end,
} }
local function stringtotable(sdata, safe) function core.deserialize(str, safe)
if sdata:byte(1) == 27 then return nil, "binary bytecode prohibited" end if str:byte(1) == 0x1B then
local f, message = assert(loadstring(sdata)) return nil, "Bytecode prohibited"
if not f then return nil, message end end
if safe then local f, err = loadstring(str)
setfenv(f, safe_env) if not f then return nil, err end
setfenv(f, safe and safe_env or env)
local good, data = pcall(f)
if good then
return data
else else
setfenv(f, env) return nil, data
end end
return f()
end end
function core.deserialize(sdata, safe)
local table = {}
local okay, results = pcall(stringtotable, sdata, safe)
if okay then
return results
end
core.log('error', 'core.deserialize(): '.. results)
return nil
end
-- Run some unit tests -- Unit tests
local function unit_test() local test_in = {cat={sound="nyan", speed=400}, dog={sound="woof"}}
function unitTest(name, success) local test_out = core.deserialize(core.serialize(test_in))
if not success then
error(name .. ': failed')
end
end
unittest_input = {cat={sound="nyan", speed=400}, dog={sound="woof"}} assert(test_in.cat.sound == test_out.cat.sound)
unittest_output = core.deserialize(core.serialize(unittest_input)) assert(test_in.cat.speed == test_out.cat.speed)
assert(test_in.dog.sound == test_out.dog.sound)
unitTest("test 1a", unittest_input.cat.sound == unittest_output.cat.sound) test_in = {escape_chars="\n\r\t\v\\\"\'", non_european="θשׁ٩∂"}
unitTest("test 1b", unittest_input.cat.speed == unittest_output.cat.speed) test_out = core.deserialize(core.serialize(test_in))
unitTest("test 1c", unittest_input.dog.sound == unittest_output.dog.sound) assert(test_in.escape_chars == test_out.escape_chars)
assert(test_in.non_european == test_out.non_european)
unittest_input = {escapechars="\n\r\t\v\\\"\'", noneuropean="θשׁ٩∂"}
unittest_output = core.deserialize(core.serialize(unittest_input))
unitTest("test 3a", unittest_input.escapechars == unittest_output.escapechars)
unitTest("test 3b", unittest_input.noneuropean == unittest_output.noneuropean)
end
unit_test() -- Run it
unit_test = nil -- Hide it