Moves sources, adds schematics and fixes map code

- The .py files are moved in a folder called 'src'
 - Schematics: Adds schematics. They are compatible with minetest's mts format. A Schematic object can import a mts file, read binary data, export its data into a BytesIO stream, and write its binary data to a file readable by minetest
 - Map: MapInterfaces can now be requested to copy a part of a map delimited by two positions in space into a Schematic object, later usable and savable. Mapblocks' nodes now show their correct position in the world. Mapblocks also store their mapblock position and the integer representing that position in the mapblock grid. MapVessels' methods' naming convention is also unified
 - Test: The picture building function is removed. Messages are added to the test functions, and a new one is implemented, removing all unknown items once provided with a map.sqlite file and another file containing all known nodes' itemstrings (dumped from the minetest server)
 - Tools: A mod was developed to be used in minetest in order to dump the known nodes list. Copy the mod and use /dumpnodes for this
This commit is contained in:
LeMagnesium 2016-02-18 23:26:55 +01:00
parent b8cbe3b75b
commit 4b8a13c4a4
13 changed files with 463 additions and 139 deletions

3
.gitignore vendored
View File

@ -1,2 +1,3 @@
__pycache__/
map.sqlite
*.sqlite
*.mts

View File

@ -16,6 +16,7 @@ from utils import *
from metadata import NodeMetaRef
from inventory import getSerializedInventory, deserializeInventory, InvRef
from nodes import NodeTimerRef, Node
from schematics import Schematic
# Bitmask constants
IS_UNDERGROUND = 1
@ -31,7 +32,9 @@ def determineMapBlock(pos):
return Pos({'x': posx, 'y': posy, 'z': posz})
class MapBlock:
def __init__(self, data = None):
def __init__(self, data = None, abspos = 0):
self.abspos = abspos
self.mapblockpos = posFromInt(self.abspos, 4096)
if data:
self.explode(data)
else:
@ -180,6 +183,12 @@ class MapBlock:
self.num_name_id_mappings = len(self.name_id_mappings)
return True
def add_node(self, mapblockpos, node):
self.set_node(self, mapblockpos, node)
def remove_node(self, mapblockpos):
self.set_node(mapblockpos, Node("air"))
def explode(self, bytelist):
data = BytesIO(bytelist)
@ -373,7 +382,7 @@ class MapBlock:
self.static_objects.append({
"type": otype,
"pos": Pos({'x': pos_x_nodes, 'y': pos_y_nodes, 'z': pos_z_nodes}),
"pos": Pos({'x': pos_x_nodes + self.mapblockpos.x, 'y': pos_y_nodes + self.mapblockpos.y, 'z': pos_z_nodes + self.mapblockpos.z}),
"data": str(odata),
})
@ -411,7 +420,11 @@ class MapBlock:
itemstring = self.name_id_mappings[node_data["param0"][id]]
param1 = node_data["param1"][id]
param2 = node_data["param2"][id]
self.nodes[id] = Node(itemstring, param1 = param1, param2 = param2, pos = posFromInt(id, self.mapblocksize))
pos = posFromInt(id, self.mapblocksize)
pos.x += self.mapblockpos.x
pos.z += self.mapblockpos.z
pos.y += self.mapblockpos.y
self.nodes[id] = Node(itemstring, param1 = param1, param2 = param2, pos = pos)
# EOF!
self.loaded = True
@ -419,7 +432,7 @@ class MapBlock:
def get_meta(self, abspos):
self.check_pos(abspos)
return self.node_meta[abspos]
return self.node_meta.get(abspos) or NodeMetaRef()
class MapVessel:
def __init__(self, mapfile, backend = "sqlite3"):
@ -483,9 +496,11 @@ class MapVessel:
try:
self.cur.execute("REPLACE INTO `blocks` (`pos`, `data`) VALUES ({0}, ?)".format(blockID),
[self.cache[blockID]])
#self.cur.execute("COMMIT")
except _sql.OperationalError as err:
raise MapError(err)
def commit(self):
self.conn.commit()
def load(self, blockID):
@ -499,7 +514,7 @@ class MapVessel:
elif not res:
return res, code
return MapBlock(self.cache[blockID])
return MapBlock(self.cache[blockID], abspos = blockID)
def store(self, blockID, mapblockData):
if self.isEmpty():
@ -521,38 +536,45 @@ class MapInterface:
self.mod_cache = []
self.force_save_on_unload = True
def modFlag(self, mapblockpos):
def mod_flag(self, mapblockpos):
if not mapblockpos in self.mod_cache:
self.mod_cache.append(mapblockpos)
def unloadMapBlock(self, blockID):
def unload_mapblock(self, blockID):
self.mapblocks[blockID] = None
del self.cache_history[self.cache_history.index(blockID)]
if self.mod_cache.index(blockID) != -1:
if blockID in self.mod_cache:
if not self.force_save_on_unload:
print("Unloading unsaved mapblock at pos {0}!".format(blockID))
del self.mod_cache[self.mod_cache.index(blockID)]
else:
print("Saving unsaved mapblock at pos {0} before unloading it.".format(blockID))
self.saveMapBlock(blockID)
self.save_mapblock(blockID)
self.interface.uncache(blockID)
def setMaxCacheSize(self, size):
def set_maxcachesize(self, size):
if type(size) != type(0):
raise TypeError("Invalid type for size: {0}".format(type(size)))
self.max_cache_size = size
self.check_cache()
def loadMapBlock(self, blockID):
def check_cache(self):
while len(self.interface.cache) > self.max_cache_size:
self.interface.uncache(self.cache_history[0])
self.unload_mapblock(self.cache_history[0])
def get_maxcachesize(self):
return self.max_cache_size
def load_mapblock(self, blockID):
self.mapblocks[blockID] = self.interface.load(blockID)
if not blockID in self.cache_history:
self.cache_history.append(blockID)
if len(self.cache_history) > self.max_cache_size:
self.interface.uncache(self.cache_history[0])
self.unloadMapBlock(self.cache_history[0])
self.check_cache()
def saveMapBlock(self, blockID):
def save_mapblock(self, blockID):
if not self.mapblocks.get(blockID):
return False
@ -564,10 +586,10 @@ class MapInterface:
def check_for_pos(self, mapblockpos):
if not self.mapblocks.get(mapblockpos):
self.loadMapBlock(mapblockpos)
self.load_mapblock(mapblockpos)
if not self.mapblocks.get(mapblockpos):
self.unloadMapBlock(mapblockpos)
self.unload_mapblock(mapblockpos)
return False
return True
@ -587,20 +609,54 @@ class MapInterface:
raise IgnoreContentReplacementError("Pos: " + pos)
node.pos = pos
self.modFlag(mapblockpos)
self.mod_flag(mapblockpos)
return self.mapblocks[mapblockpos].set_node((pos.x % 16) + (pos.y % 16) * 16 + (pos.z % 16) * 16 * 16, node)
def remove_node(self, pos):
mapblock = determineMapBlock(pos)
mapblockpos = getMapBlockPos(mapblock)
if not self.check_for_pos(mapblockpos):
return
return self.mapblocks[mapblockpos].remove_node((pos.x % 16) + (pos.y % 16) * 16 + (pos.z % 16) * 16 * 16, node)
def save(self):
while len(self.mod_cache) > 0:
self.saveMapBlock(self.mod_cache[0])
self.save_mapblock(self.mod_cache[0])
self.mod_cache = []
self.interface.commit()
def get_meta(self, pos):
mapblock = determineMapBlock(pos)
mapblockpos = getMapBlockPos(mapblock)
self.modFlag(mapblockpos)
self.mod_flag(mapblockpos)
if not self.check_for_pos(mapblockpos):
return NodeMetaRef()
return self.mapblocks[mapblockpos].get_meta(intFromPos(pos, 16))
# The schematics stuff
def export_schematic(self, startpos, endpos, forceplace = True):
# Get the corners first
spos = Pos({"x": min(startpos.x, endpos.x), "y": min(startpos.y, endpos.y), "z": min(startpos.z, endpos.z)})
epos = Pos({"x": max(startpos.x, endpos.x), "y": max(startpos.y, endpos.y), "z": max(startpos.z, endpos.z)})
schem = {}
schem["size"] = {"x": epos.x - spos.x, "y": epos.y - spos.y, "z": epos.z - spos.y}
schem["data"] = {}
for x in range(schem["size"]["x"]):
for y in range(schem["size"]["y"]):
for z in range(schem["size"]["z"]):
schem["data"][x + (y * schem["size"]["x"]) + (z * schem["size"]["y"] * schem["size"]["x"])] = {
"name": self.get_node(Pos({"x": spos.x + x, "y": spos.y + y, "z": spos.z + z})).get_name(),
"prob": 255,
"force_place": forceplace
}
sch = Schematic()
sch.serialize_schematic(schem)
return sch

View File

@ -48,3 +48,24 @@ class Node:
def get_name(self):
return self.itemstring
def get_param1(self):
return self.param1
def get_param2(self):
return self.param2
def get_pos(self):
return self.pos
def set_name(self, name):
self.itemstring = name
def set_param1(self, param):
self.param1 = param
def set_param2(self, param):
self.param2 = param
def set_pos(self, pos):
self.pos = pos

191
src/schematics.py Normal file
View File

@ -0,0 +1,191 @@
#!/usr/bin/env python3
# -*- encoding: utf8 -*-
###########################
## Schematics for Python-MT
##
##
#
from nodes import Node
from utils import readU16, readU8, readU32, writeU16, writeU8, writeU32
import zlib
from io import BytesIO
# See :
# https://github.com/minetest/minetest/blob/master/src/mg_schematic.cpp#L339
# https://github.com/minetest/minetest/blob/master/src/mapnode.cpp#L548
# https://github.com/minetest/minetest/blob/master/src/mg_schematic.cpp#L260
# Quick spec for ver4
"""
u32 signature (= b"MTSM")
u16 version?
u16 size_x
u16 size_y
u16 size_z
foreach size_y:
u8 slice_prob
u16 num_names
foreach num_names:
u16 name_len
u8[name_len] name
zlib encrypted mapnode bulk data (see src/mapnode.cpp)
foreach size_x * size_y * size_z:
u16 param0
foreach size_x * size_y * size_z:
u8 param1
foreach size_x * size_y * size_z:
u8 param2
"""
# Definitions
class Schematic:
def __init__(self, filename = None):
self.filename = filename
self.loaded = False
self._init_data()
if self.filename:
self.load_from_file(filename)
def _init_data(self):
self.version = -1
self.size = {}
self.y_slice_probs = {}
self.nodes = []
self.data = {}
def load(self, data):
self._init_data()
self.loaded = False
try:
assert(data.read(4) == b"MTSM")
except AssertionError:
print("ERROR: {0} couldn't load schematic from data : invalid signature".format(self))
return
self.version = readU16(data)
self.size = {"x": readU16(data), "y": readU16(data), "z": readU16(data)}
for i in range(self.size["y"]):
p = readU8(data)
if p < 127:
self.y_slice_probs[i] = p
for _ in range(readU16(data)):
nodename = ""
for _ in range(readU16(data)):
nodename += chr(readU8(data))
self.nodes.append(nodename)
bulk = BytesIO(zlib.decompress(data.read()))
nodecount = self.size["x"] * self.size["y"] * self.size["z"]
self.data = {}
for i in range(nodecount):
self.data[i] = Node(self.nodes[readU16(bulk)])
for i in range(nodecount):
self.data[i].set_param1(readU8(bulk))
for i in range(nodecount):
self.data[i].set_param2(readU8(bulk))
self.loaded = True
def export(self):
if not self.loaded:
return
data = BytesIO(b"")
data.write(b"MTSM")
writeU16(data, self.version)
writeU8(data, self.size["x"])
writeU8(data, self.size["y"])
writeU8(data, self.size["z"])
for u in range(self.size["y"]):
p = self.y_slice_probs.get(u) or 127
writeU8(data, p)
writeU16(data, len(self.nodes))
for node in self.nodes:
writeU16(data, len(node))
for c in node:
writeU8(data, ord(c))
bulk = BytesIO(b"")
nodecount = self.size["x"] * self.size["y"] * self.size["z"]
for i in range(nodecount):
writeU16(bulk, self.nodes.index(self.data[i].get_name()))
for i in range(nodecount):
writeU8(bulk, self.data[i].get_param1())
for i in range(nodecount):
writeU8(bulk, self.data[i].get_param2())
bulk.seek(0)
data.write(zlib.compress(bulk.read()))
data.seek(0)
return data
def load_from_file(self, filename):
try:
ifile = open(filename, "rb")
except Exception as err:
print("ERROR: {0} couldn't open file {1} : {2}".format(self, filename, err))
return
self.load(ifile)
def export_to_file(self, filename):
try:
ofile = open(filename, "wb")
except Exception as err:
print("ERROR: {0} couldn't open file {1} : {2}".format(self, filename, err))
return
ofile.write(self.export().read())
def serialize_schematic(self, schemtab):
self._init_data()
self.version = 4
self.size = schemtab["size"]
if schemtab.get("y_slice_probs"):
for prob in schemtab["y_slice_probs"]:
self.y_slice_probs[prob[0]] = prob[1]
for index in schemtab["data"]:
entry = schemtab["data"][index]
if not entry["name"] in self.nodes:
self.nodes.append(entry["name"])
self.data[index] = Node(entry["name"], param1 = entry["prob"], param2 = entry.get("param2") or 0)
if not entry.get("force_place"):
self.data[index].set_param1(int(entry["prob"] / 2))
self.loaded = True
def get_node(self, pos):
if not self.loaded:
return
if pos.x > self.size["x"] or pos.y > self.size["y"] or pos.z > self.size["z"]:
return
abspos = pos.x + (pos.y * self.size["x"]) + (pos.z * self.size["y"] * self.size["x"])
return self.data[abspos]

150
src/test.py Executable file
View File

@ -0,0 +1,150 @@
#!/usr/bin/env python3.4
# -*- encoding: utf-8 -*-
############################
## Tests ran for Python-MT
##
import minetest
import random
from utils import readS8, readS16, Pos
from io import BytesIO
from schematics import Schematic
def testSignedEndians():
print(readS16(BytesIO(b"\x9f\xff")))
def testMapBlockLoad():
file = minetest.map.MapVessel("./map.sqlite")
#i = 2+12*4096+(-9)*4096*4096
#i = 0
for i in range(-4096, 4096):
res, code = file.read(i)
if not res:
continue
else:
print("Read {0}: {1}".format(i, (res, code)))
mapb = file.load(i)
print("Loaded {0}: {1}".format(i, mapb))
print(len(file.cache))
def testGetNode():
db = minetest.MapInterface("./map.sqlite")
for _ in range(1000):
pos = Pos({'x': random.randint(-300, 300), 'y': random.randint(-300, 300), 'z': random.randint(-300, 300)})
print("{0}: {1}".format(pos, db.get_node(pos).get_name()))
print("Cache size: {0}".format(len(db.interface.cache)), end = " \r")
assert(len(db.interface.cache) <= db.get_maxcachesize())
def testSetNode():
db = minetest.MapInterface("./map.sqlite")
f = open("./dump.bin", "w")
dummy = minetest.Node("default:nyancat")
for y in range(1, 10):
db.set_node(Pos({'x': 0, 'y': y, 'z': 0}), dummy)
db.save()
def invManip():
db = minetest.MapInterface("./map.sqlite")
chest = db.get_meta(Pos({'x': 0, 'y': 0, 'z': 0}))
#print(chest)
inv = chest.get_inventory()
#print(inv)
#print(chest.get_string("formspec"))
#chest.set_string("formspec", chest.get_string("formspec") + "button[0,0;1,0.5;moo;Moo]")
#print(chest.get_string("formspec"))
print(inv.is_empty("main"))
print(inv.get_size("main"))
db.save()
def testSchematics():
# Import from file
schem = Schematic("/home/lymkwi/.minetest/games/minetest_game/mods/default/schematics/apple_tree_from_sapling.mts")
# Export to BytesStream & file
print(schem.export().read())
schem.export_to_file("test.mts")
# Map export
db = minetest.MapInterface("./map.sqlite")
schem = db.export_schematic(Pos(), Pos({"x": -100, "y": 10, "z": 100}))
schem.export_to_file("test.mts")
# Get node
print(schem.get_node(Pos({"x": 1, "y": 1, "z": 0})))
def removeUnknowns():
import sys
if len(sys.argv) < 2:
print("I need a map.sqlite file!")
return
elif len(sys.argv) < 3:
print("I need a known nodes file!")
return
try:
knodes = open(sys.argv[2])
except Exception as err:
print("Couldn't open know nodes file {0} : {1}".format(sys.argv[1], err))
return
print("Know nodes file opened")
nodes = [node[:-1] for node in knodes.readlines()] # Remove the \n
print("{0} nodes known".format(len(nodes)))
u = minetest.map.MapVessel(sys.argv[1])
ma = 4096 + 4096 * 4096# + 4096 * 4096 * 4096
for i in range(-ma, ma):
k = u.load(i)
absi = minetest.utils.posFromInt(i, 4096)
print("Testing mapblock {0} ({1}) ".format(i, absi), end = '\r')
if k:
print("Checking mapblock {0} ({1})".format(i, absi), end = " \r")
unknowns = []
for id in k.name_id_mappings:
node = k.name_id_mappings[id]
if node != "air":
if not node in nodes:
print("Unknown node in {0} : {1}".format(i, node))
unknowns.append(node)
if len(unknowns) > 0:
for x in range(16):
for y in range(16):
for z in range(16):
noderef = k.get_node(x + y * 16 + z * 16 * 16)
if noderef.get_name() in unknowns:
print("Removed node in {0} : {1}".format(noderef.get_pos(), noderef.get_name()))
k.remove_node(x + y * 16 + z * 16 * 16)
print("Saving mapblock {0}".format(absi))
u.store(i, k.implode())
u.write(i)
u.uncache(i)
u.commit()
if __name__ == "__main__":
#findTheShelves()
print("=> MapBlockLoad")
#testMapBlockLoad()
removeUnknowns()
"""print("=> signed endians")
testSignedEndians()
print("=> get_node")
testGetNode()
print("=> set_node")
testSetNode()
print("=> inventory manipulation (WIP)")
invManip()
print("=> schematic manipulation (WIP)")
testSchematics()"""

View File

@ -138,4 +138,4 @@ def writeU32(strm, val):
val -= k
val /= 256
strm.write(bytes(vals))
strm.write(bytes(vals))

116
test.py
View File

@ -1,116 +0,0 @@
#!/usr/bin/env python3.4
# -*- encoding: utf-8 -*-
############################
## Tests ran for Python-MT
##
import minetest
import random
from utils import readS8, readS16, Pos
from io import BytesIO
def findTheShelves():
from PIL import Image
file = minetest.MapInterface("/home/lymkwi/.minetest/worlds/NodesJustWannaHaveFun/map.sqlite")
file.setMaxCacheSize(450)
mapy = {}
size = 16 # In mapblocks
hsize = int(size/2)
alphas, noairs = {}, {}
for x in range(-hsize, hsize):
for z in range(-hsize, hsize):
for y in range(0, 16):
posx, posy, posz = x*16, y*16, z*16
for intx in range(0, 16):
for intz in range(0, 16):
alpha, noair = 0, 0
for inty in range(0, 16):
node = file.get_node(Pos({'x': posx+intx, 'y': posy+inty, 'z': posz+intz}))
print("[{0}] {1}".format(node.pos, node.get_name()), end = (' ' * 20) + '\r')
if node.get_name() != "air":
noair += 1
if node.get_name() == "ignore":
alpha += 1
coords = ((posx + intx) * 4096 * (posz + intz))
alphas[coords] = (alphas.get(coords) or 256) - alpha
noairs[coords] = (noairs.get(coords) or 0) + noair
# mapy = [(alpha,) * 3 + (noair,) ]
for alpha, noair in zip(alphas.keys(), noairs.keys()):
mapy[alpha] = (alphas[alpha],) * 3 + (noairs[noair],)
buildPic(mapy, size).save("map.jpg")
def buildPic(mapy, size):
im = Image.new("RGBA", (size*16, size*16))
hsize = int(size*8)
for x in range(-hsize, hsize):
for z in range(-hsize, hsize):
im.putpixel((z+hsize, x+hsize), mapy[x * z*4096])
im.show()
return im
def testSignedEndians():
print(readS16(BytesIO(b"\x9f\xff")))
def testMapBlockLoad():
file = minetest.map.MapVessel("./map.sqlite")
#i = 2+12*4096+(-9)*4096*4096
#i = 0
for i in range(-4096, 4096):
res, code = file.read(i)
if not res:
continue
else:
print("Read {0}: {1}".format(i, (res, code)))
mapb = file.load(i)
print("Loaded {0}: {1}".format(i, mapb))
print(len(file.cache))
def testGetNode():
db = minetest.MapInterface("./map.sqlite")
for _ in range(1000):
pos = Pos({'x': random.randint(-300, 300), 'y': random.randint(-300, 300), 'z': random.randint(-300, 300)})
print("{0}: {1}".format(pos, db.get_node(pos).get_name()))
print("Cache: {0}".format(len(db.interface.cache)))
def testSetNode():
db = minetest.MapInterface("./map.sqlite")
f = open("./dump.bin", "w")
dummy = minetest.Node("default:nyancat")
for y in range(1, 256):
db.set_node(Pos({'x': 0, 'y': y, 'z': 0}), dummy)
db.save()
def invManip():
db = minetest.MapInterface("./map.sqlite")
chest = db.get_meta(Pos({'x': 0, 'y': 0, 'z': 0}))
#print(chest)
inv = chest.get_inventory()
#print(inv)
#print(chest.get_string("formspec"))
#chest.set_string("formspec", chest.get_string("formspec") + "button[0,0;1,0.5;moo;Moo]")
#print(chest.get_string("formspec"))
print(inv.is_empty("main"))
print(inv.get_size("main"))
db.save()
if __name__ == "__main__":
#findTheShelves()
#testMapBlockLoad()
#testSignedEndians()
#testGetNode()
#testSetNode()
invManip()

View File

@ -0,0 +1,16 @@
minetest.register_chatcommand("dumpnodes", {
privs = {server = true},
description = "Dumps all known nodes in a file",
func = function()
local f = io.open(minetest.get_modpath("devel") .. "/knownnodes.txt", "w")
for node in pairs(minetest.registered_nodes) do
f:write(node)
f:write('\n')
end
f:flush()
f:close()
return true, "Nodes dumped in " .. minetest.get_modpath("devel") .. "/knownnodes.txt"
end
})

View File

@ -0,0 +1,5 @@
-- Mod to use for python-minetest
-- License : WTFPL
-- By Mg/LeMagnesium
dofile(minetest.get_modpath("devel") .. "/dumpnodes.lua")