Implement AreaStore serialization

master
ShadowNinja 2015-10-30 19:27:48 -04:00
parent c4b7afed7e
commit 821551a266
6 changed files with 104 additions and 95 deletions

View File

@ -2737,6 +2737,10 @@ If you chose the parameter-less constructor, a fast implementation will be autom
block_radius = number, -- the radius (in nodes) of the areas the cache generates prefiltered lists for, minimum 16, default 64 block_radius = number, -- the radius (in nodes) of the areas the cache generates prefiltered lists for, minimum 16, default 64
limit = number, -- the cache's size, minimum 20, default 1000 limit = number, -- the cache's size, minimum 20, default 1000
} }
* `to_string()`: Experimental. Returns area store serialized as a (binary) string.
* `to_file(filename)`: Experimental. Like `to_string()`, but writes the data to a file.
* `from_string(str)`: Experimental. Deserializes string and loads it into the AreaStore. Returns success and, optionally, an error message.
* `from_file(filename)`: Experimental. Like `from_string()`, but reads the data from a file.
### `ItemStack` ### `ItemStack`
An `ItemStack` is a stack of items. An `ItemStack` is a stack of items.

View File

@ -70,6 +70,22 @@ static inline void push_areas(lua_State *L, const std::vector<Area *> &areas,
} }
} }
// Deserializes value and handles errors
static int deserialization_helper(lua_State *L, AreaStore *as,
std::istream &is)
{
try {
as->deserialize(is);
} catch (const SerializationError &e) {
lua_pushboolean(L, false);
lua_pushstring(L, e.what());
return 2;
}
lua_pushboolean(L, true);
return 1;
}
// garbage collector // garbage collector
int LuaAreaStore::gc_object(lua_State *L) int LuaAreaStore::gc_object(lua_State *L)
{ {
@ -217,17 +233,15 @@ int LuaAreaStore::l_set_cache_params(lua_State *L)
return 0; return 0;
} }
#if 0
// to_string() // to_string()
int LuaAreaStore::l_to_string(lua_State *L) int LuaAreaStore::l_to_string(lua_State *L)
{ {
NO_MAP_LOCK_REQUIRED; NO_MAP_LOCK_REQUIRED;
LuaAreaStore *o = checkobject(L, 1); LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;
std::ostringstream os(std::ios_base::binary); std::ostringstream os(std::ios_base::binary);
ast->serialize(os); o->as->serialize(os);
std::string str = os.str(); std::string str = os.str();
lua_pushlstring(L, str.c_str(), str.length()); lua_pushlstring(L, str.c_str(), str.length());
@ -258,16 +272,12 @@ int LuaAreaStore::l_from_string(lua_State *L)
NO_MAP_LOCK_REQUIRED; NO_MAP_LOCK_REQUIRED;
LuaAreaStore *o = checkobject(L, 1); LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;
size_t len; size_t len;
const char *str = luaL_checklstring(L, 2, &len); const char *str = luaL_checklstring(L, 2, &len);
std::istringstream is(std::string(str, len), std::ios::binary); std::istringstream is(std::string(str, len), std::ios::binary);
bool success = ast->deserialize(is); return deserialization_helper(L, o->as, is);
lua_pushboolean(L, success);
return 1;
} }
// from_file(filename) // from_file(filename)
@ -276,18 +286,13 @@ int LuaAreaStore::l_from_file(lua_State *L)
NO_MAP_LOCK_REQUIRED; NO_MAP_LOCK_REQUIRED;
LuaAreaStore *o = checkobject(L, 1); LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;
const char *filename = luaL_checkstring(L, 2); const char *filename = luaL_checkstring(L, 2);
CHECK_SECURE_PATH_OPTIONAL(L, filename); CHECK_SECURE_PATH_OPTIONAL(L, filename);
std::ifstream is(filename, std::ios::binary); std::ifstream is(filename, std::ios::binary);
bool success = ast->deserialize(is); return deserialization_helper(L, o->as, is);
lua_pushboolean(L, success);
return 1;
} }
#endif
LuaAreaStore::LuaAreaStore() LuaAreaStore::LuaAreaStore()
{ {
@ -377,9 +382,9 @@ const luaL_reg LuaAreaStore::methods[] = {
luamethod(LuaAreaStore, reserve), luamethod(LuaAreaStore, reserve),
luamethod(LuaAreaStore, remove_area), luamethod(LuaAreaStore, remove_area),
luamethod(LuaAreaStore, set_cache_params), luamethod(LuaAreaStore, set_cache_params),
/* luamethod(LuaAreaStore, to_string), luamethod(LuaAreaStore, to_string),
luamethod(LuaAreaStore, to_file), luamethod(LuaAreaStore, to_file),
luamethod(LuaAreaStore, from_string), luamethod(LuaAreaStore, from_string),
luamethod(LuaAreaStore, from_file),*/ luamethod(LuaAreaStore, from_file),
{0,0} {0,0}
}; };

View File

@ -43,11 +43,11 @@ private:
static int l_set_cache_params(lua_State *L); static int l_set_cache_params(lua_State *L);
/* static int l_to_string(lua_State *L); static int l_to_string(lua_State *L);
static int l_to_file(lua_State *L); static int l_to_file(lua_State *L);
static int l_from_string(lua_State *L); static int l_from_string(lua_State *L);
static int l_from_file(lua_State *L); */ static int l_from_file(lua_State *L);
public: public:
AreaStore *as; AreaStore *as;

View File

@ -31,6 +31,7 @@ public:
void genericStoreTest(AreaStore *store); void genericStoreTest(AreaStore *store);
void testVectorStore(); void testVectorStore();
void testSpatialStore(); void testSpatialStore();
void testSerialization();
}; };
static TestAreaStore g_test_instance; static TestAreaStore g_test_instance;
@ -41,6 +42,7 @@ void TestAreaStore::runTests(IGameDef *gamedef)
#if USE_SPATIAL #if USE_SPATIAL
TEST(testSpatialStore); TEST(testSpatialStore);
#endif #endif
TEST(testSerialization);
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
@ -121,3 +123,40 @@ void TestAreaStore::genericStoreTest(AreaStore *store)
store->removeArea(d.id); store->removeArea(d.id);
} }
void TestAreaStore::testSerialization()
{
VectorAreaStore store;
Area a(v3s16(-1, 0, 1), v3s16(0, 1, 2));
a.data = "Area A";
store.insertArea(&a);
Area b(v3s16(123, 456, 789), v3s16(32000, 100, 10));
b.data = "Area B";
store.insertArea(&b);
std::ostringstream os;
store.serialize(os);
std::string str = os.str();
std::string str_wanted("\x00" // Version
"\x00\x02" // Count
"\xFF\xFF\x00\x00\x00\x01" // Area A min edge
"\x00\x00\x00\x01\x00\x02" // Area A max edge
"\x00\x06" // Area A data length
"Area A" // Area A data
"\x00\x7B\x00\x64\x00\x0A" // Area B min edge (last two swapped with max edge for sorting)
"\x7D\x00\x01\xC8\x03\x15" // Area B max edge (^)
"\x00\x06" // Area B data length
"Area B", // Area B data
1 + 2 +
6 + 6 + 2 + 6 +
6 + 6 + 2 + 6);
UASSERTEQ(std::string, str, str_wanted);
std::istringstream is(str);
store.deserialize(is);
UASSERTEQ(size_t, store.size(), 4); // deserialize() doesn't clear the store
}

View File

@ -62,56 +62,41 @@ const Area *AreaStore::getArea(u32 id) const
return &it->second; return &it->second;
} }
#if 0
Currently, serialisation is commented out. This is because of multiple reasons:
1. Why do we store the areastore into a file, why not into the database?
2. We don't use libspatial's serialisation, but we should, or perhaps not, because
it would remove the ability to switch. Perhaps write migration routines?
3. Various things need fixing, e.g. the size is serialized as
c++ implementation defined size_t
bool AreaStore::deserialize(std::istream &is)
{
u8 ver = readU8(is);
if (ver != 1)
return false;
u16 count_areas = readU16(is);
for (u16 i = 0; i < count_areas; i++) {
// deserialize an area
Area a;
a.id = readU32(is);
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
a.datalen = readU16(is);
a.data = new char[a.datalen];
is.read((char *) a.data, a.datalen);
insertArea(a);
}
return true;
}
static bool serialize_area(void *ostr, Area *a)
{
std::ostream &os = *((std::ostream *) ostr);
writeU32(os, a->id);
writeV3S16(os, a->minedge);
writeV3S16(os, a->maxedge);
writeU16(os, a->datalen);
os.write(a->data, a->datalen);
return false;
}
void AreaStore::serialize(std::ostream &os) const void AreaStore::serialize(std::ostream &os) const
{ {
// write initial data writeU8(os, 0); // Serialisation version
writeU8(os, 1); // serialisation version
writeU16(os, areas_map.size()); //DANGER: not platform independent // TODO: Compression?
forEach(&serialize_area, &os); writeU16(os, areas_map.size());
for (AreaMap::const_iterator it = areas_map.begin();
it != areas_map.end(); ++it) {
const Area &a = it->second;
writeV3S16(os, a.minedge);
writeV3S16(os, a.maxedge);
writeU16(os, a.data.size());
os.write(a.data.data(), a.data.size());
}
} }
#endif void AreaStore::deserialize(std::istream &is)
{
u8 ver = readU8(is);
if (ver != 0)
throw SerializationError("Unknown AreaStore "
"serialization version!");
u16 num_areas = readU16(is);
for (u32 i = 0; i < num_areas; ++i) {
Area a;
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
u16 data_len = readU16(is);
char *data = new char[data_len];
is.read(data, data_len);
a.data = std::string(data, data_len);
insertArea(&a);
}
}
void AreaStore::invalidateCache() void AreaStore::invalidateCache()
{ {
@ -226,18 +211,6 @@ void VectorAreaStore::getAreasInArea(std::vector<Area *> *result,
} }
} }
#if 0
bool SimpleAreaStore::forEach(ForEachCallback callback, void *arg) const
{
for (size_t i = 0; i < m_areas.size(); ++i) {
if (callback(m_areas[i], arg)) {
return true;
}
}
return false;
}
#endif
#if USE_SPATIAL #if USE_SPATIAL
static inline SpatialIndex::Region get_spatial_region(const v3s16 minedge, static inline SpatialIndex::Region get_spatial_region(const v3s16 minedge,
@ -301,14 +274,6 @@ void SpatialAreaStore::getAreasInArea(std::vector<Area *> *result,
} }
} }
#if 0
bool SpatialAreaStore::forEach(ForEachCallback callback, void *arg) const
{
// TODO ?? (this is only needed for serialisation, but libspatial has its own serialisation)
return false;
}
#endif
SpatialAreaStore::~SpatialAreaStore() SpatialAreaStore::~SpatialAreaStore()
{ {
delete m_tree; delete m_tree;

View File

@ -92,16 +92,14 @@ public:
/// or NULL if it doesn't exist. /// or NULL if it doesn't exist.
const Area *getArea(u32 id) const; const Area *getArea(u32 id) const;
#if 0 /// Serializes the store's areas to a binary ostream.
typedef bool (*ForEachCallback)(const Area *a, void *arg);
/// Calls a passed function for every stored area, until the
/// callback returns true. If that happens, it returns true,
/// if the search is exhausted, it returns false.
virtual bool forEach(ForEachCallback, void *arg=NULL) const = 0;
void serialize(std::ostream &is) const; void serialize(std::ostream &is) const;
bool deserialize(std::istream &is);
#endif /// Deserializes the Areas from a binary istream.
/// This does not currently clear the AreaStore before adding the
/// areas, making it possible to deserialize multiple serialized
/// AreaStores.
void deserialize(std::istream &is);
protected: protected:
/// Invalidates the getAreasForPos cache. /// Invalidates the getAreasForPos cache.
@ -141,7 +139,6 @@ public:
virtual bool removeArea(u32 id); virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result, virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap); v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;
protected: protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos); virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);
@ -162,7 +159,6 @@ public:
virtual bool removeArea(u32 id); virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result, virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap); v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;
protected: protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos); virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);