diff --git a/.gitignore b/.gitignore index 97d90e5..16ba523 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/ -map.sqlite +*.sqlite +*.mts diff --git a/errors.py b/src/errors.py similarity index 100% rename from errors.py rename to src/errors.py diff --git a/inventory.py b/src/inventory.py similarity index 100% rename from inventory.py rename to src/inventory.py diff --git a/map.py b/src/map.py similarity index 86% rename from map.py rename to src/map.py index 3020cb5..c8e5c9d 100644 --- a/map.py +++ b/src/map.py @@ -16,6 +16,7 @@ from utils import * from metadata import NodeMetaRef from inventory import getSerializedInventory, deserializeInventory, InvRef from nodes import NodeTimerRef, Node +from schematics import Schematic # Bitmask constants IS_UNDERGROUND = 1 @@ -31,7 +32,9 @@ def determineMapBlock(pos): return Pos({'x': posx, 'y': posy, 'z': posz}) class MapBlock: - def __init__(self, data = None): + def __init__(self, data = None, abspos = 0): + self.abspos = abspos + self.mapblockpos = posFromInt(self.abspos, 4096) if data: self.explode(data) else: @@ -180,6 +183,12 @@ class MapBlock: self.num_name_id_mappings = len(self.name_id_mappings) return True + def add_node(self, mapblockpos, node): + self.set_node(self, mapblockpos, node) + + def remove_node(self, mapblockpos): + self.set_node(mapblockpos, Node("air")) + def explode(self, bytelist): data = BytesIO(bytelist) @@ -373,7 +382,7 @@ class MapBlock: self.static_objects.append({ "type": otype, - "pos": Pos({'x': pos_x_nodes, 'y': pos_y_nodes, 'z': pos_z_nodes}), + "pos": Pos({'x': pos_x_nodes + self.mapblockpos.x, 'y': pos_y_nodes + self.mapblockpos.y, 'z': pos_z_nodes + self.mapblockpos.z}), "data": str(odata), }) @@ -411,7 +420,11 @@ class MapBlock: itemstring = self.name_id_mappings[node_data["param0"][id]] param1 = node_data["param1"][id] param2 = node_data["param2"][id] - self.nodes[id] = Node(itemstring, param1 = param1, param2 = param2, pos = posFromInt(id, self.mapblocksize)) + pos = posFromInt(id, self.mapblocksize) + pos.x += self.mapblockpos.x + pos.z += self.mapblockpos.z + pos.y += self.mapblockpos.y + self.nodes[id] = Node(itemstring, param1 = param1, param2 = param2, pos = pos) # EOF! self.loaded = True @@ -419,7 +432,7 @@ class MapBlock: def get_meta(self, abspos): self.check_pos(abspos) - return self.node_meta[abspos] + return self.node_meta.get(abspos) or NodeMetaRef() class MapVessel: def __init__(self, mapfile, backend = "sqlite3"): @@ -483,9 +496,11 @@ class MapVessel: try: self.cur.execute("REPLACE INTO `blocks` (`pos`, `data`) VALUES ({0}, ?)".format(blockID), [self.cache[blockID]]) - #self.cur.execute("COMMIT") + except _sql.OperationalError as err: raise MapError(err) + + def commit(self): self.conn.commit() def load(self, blockID): @@ -499,7 +514,7 @@ class MapVessel: elif not res: return res, code - return MapBlock(self.cache[blockID]) + return MapBlock(self.cache[blockID], abspos = blockID) def store(self, blockID, mapblockData): if self.isEmpty(): @@ -521,38 +536,45 @@ class MapInterface: self.mod_cache = [] self.force_save_on_unload = True - def modFlag(self, mapblockpos): + def mod_flag(self, mapblockpos): if not mapblockpos in self.mod_cache: self.mod_cache.append(mapblockpos) - def unloadMapBlock(self, blockID): + def unload_mapblock(self, blockID): self.mapblocks[blockID] = None del self.cache_history[self.cache_history.index(blockID)] - if self.mod_cache.index(blockID) != -1: + if blockID in self.mod_cache: if not self.force_save_on_unload: print("Unloading unsaved mapblock at pos {0}!".format(blockID)) del self.mod_cache[self.mod_cache.index(blockID)] else: print("Saving unsaved mapblock at pos {0} before unloading it.".format(blockID)) - self.saveMapBlock(blockID) + self.save_mapblock(blockID) self.interface.uncache(blockID) - def setMaxCacheSize(self, size): + def set_maxcachesize(self, size): if type(size) != type(0): raise TypeError("Invalid type for size: {0}".format(type(size))) self.max_cache_size = size + self.check_cache() - def loadMapBlock(self, blockID): + def check_cache(self): + while len(self.interface.cache) > self.max_cache_size: + self.interface.uncache(self.cache_history[0]) + self.unload_mapblock(self.cache_history[0]) + + def get_maxcachesize(self): + return self.max_cache_size + + def load_mapblock(self, blockID): self.mapblocks[blockID] = self.interface.load(blockID) if not blockID in self.cache_history: self.cache_history.append(blockID) - if len(self.cache_history) > self.max_cache_size: - self.interface.uncache(self.cache_history[0]) - self.unloadMapBlock(self.cache_history[0]) + self.check_cache() - def saveMapBlock(self, blockID): + def save_mapblock(self, blockID): if not self.mapblocks.get(blockID): return False @@ -564,10 +586,10 @@ class MapInterface: def check_for_pos(self, mapblockpos): if not self.mapblocks.get(mapblockpos): - self.loadMapBlock(mapblockpos) + self.load_mapblock(mapblockpos) if not self.mapblocks.get(mapblockpos): - self.unloadMapBlock(mapblockpos) + self.unload_mapblock(mapblockpos) return False return True @@ -587,20 +609,54 @@ class MapInterface: raise IgnoreContentReplacementError("Pos: " + pos) node.pos = pos - self.modFlag(mapblockpos) + self.mod_flag(mapblockpos) return self.mapblocks[mapblockpos].set_node((pos.x % 16) + (pos.y % 16) * 16 + (pos.z % 16) * 16 * 16, node) + def remove_node(self, pos): + mapblock = determineMapBlock(pos) + mapblockpos = getMapBlockPos(mapblock) + if not self.check_for_pos(mapblockpos): + return + + return self.mapblocks[mapblockpos].remove_node((pos.x % 16) + (pos.y % 16) * 16 + (pos.z % 16) * 16 * 16, node) + def save(self): while len(self.mod_cache) > 0: - self.saveMapBlock(self.mod_cache[0]) + self.save_mapblock(self.mod_cache[0]) self.mod_cache = [] + self.interface.commit() + def get_meta(self, pos): mapblock = determineMapBlock(pos) mapblockpos = getMapBlockPos(mapblock) - self.modFlag(mapblockpos) + self.mod_flag(mapblockpos) if not self.check_for_pos(mapblockpos): return NodeMetaRef() return self.mapblocks[mapblockpos].get_meta(intFromPos(pos, 16)) + + # The schematics stuff + def export_schematic(self, startpos, endpos, forceplace = True): + + # Get the corners first + spos = Pos({"x": min(startpos.x, endpos.x), "y": min(startpos.y, endpos.y), "z": min(startpos.z, endpos.z)}) + epos = Pos({"x": max(startpos.x, endpos.x), "y": max(startpos.y, endpos.y), "z": max(startpos.z, endpos.z)}) + + schem = {} + schem["size"] = {"x": epos.x - spos.x, "y": epos.y - spos.y, "z": epos.z - spos.y} + schem["data"] = {} + for x in range(schem["size"]["x"]): + for y in range(schem["size"]["y"]): + for z in range(schem["size"]["z"]): + schem["data"][x + (y * schem["size"]["x"]) + (z * schem["size"]["y"] * schem["size"]["x"])] = { + "name": self.get_node(Pos({"x": spos.x + x, "y": spos.y + y, "z": spos.z + z})).get_name(), + "prob": 255, + "force_place": forceplace + } + + sch = Schematic() + sch.serialize_schematic(schem) + + return sch diff --git a/metadata.py b/src/metadata.py similarity index 100% rename from metadata.py rename to src/metadata.py diff --git a/minetest.py b/src/minetest.py similarity index 100% rename from minetest.py rename to src/minetest.py diff --git a/nodes.py b/src/nodes.py similarity index 73% rename from nodes.py rename to src/nodes.py index 3e34309..f9b4b1f 100644 --- a/nodes.py +++ b/src/nodes.py @@ -48,3 +48,24 @@ class Node: def get_name(self): return self.itemstring + + def get_param1(self): + return self.param1 + + def get_param2(self): + return self.param2 + + def get_pos(self): + return self.pos + + def set_name(self, name): + self.itemstring = name + + def set_param1(self, param): + self.param1 = param + + def set_param2(self, param): + self.param2 = param + + def set_pos(self, pos): + self.pos = pos diff --git a/src/schematics.py b/src/schematics.py new file mode 100644 index 0000000..1236df4 --- /dev/null +++ b/src/schematics.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Schematics for Python-MT +## +## +# + +from nodes import Node +from utils import readU16, readU8, readU32, writeU16, writeU8, writeU32 + +import zlib +from io import BytesIO + +# See : +# https://github.com/minetest/minetest/blob/master/src/mg_schematic.cpp#L339 +# https://github.com/minetest/minetest/blob/master/src/mapnode.cpp#L548 +# https://github.com/minetest/minetest/blob/master/src/mg_schematic.cpp#L260 + +# Quick spec for ver4 +""" +u32 signature (= b"MTSM") +u16 version? +u16 size_x +u16 size_y +u16 size_z + +foreach size_y: + u8 slice_prob + +u16 num_names +foreach num_names: + u16 name_len + u8[name_len] name + +zlib encrypted mapnode bulk data (see src/mapnode.cpp) +foreach size_x * size_y * size_z: + u16 param0 + +foreach size_x * size_y * size_z: + u8 param1 + +foreach size_x * size_y * size_z: + u8 param2 +""" + +# Definitions + +class Schematic: + def __init__(self, filename = None): + self.filename = filename + self.loaded = False + self._init_data() + if self.filename: + self.load_from_file(filename) + + def _init_data(self): + self.version = -1 + self.size = {} + self.y_slice_probs = {} + self.nodes = [] + self.data = {} + + def load(self, data): + self._init_data() + self.loaded = False + + try: + assert(data.read(4) == b"MTSM") + except AssertionError: + print("ERROR: {0} couldn't load schematic from data : invalid signature".format(self)) + return + + self.version = readU16(data) + self.size = {"x": readU16(data), "y": readU16(data), "z": readU16(data)} + + for i in range(self.size["y"]): + p = readU8(data) + if p < 127: + self.y_slice_probs[i] = p + + for _ in range(readU16(data)): + nodename = "" + for _ in range(readU16(data)): + nodename += chr(readU8(data)) + self.nodes.append(nodename) + + + bulk = BytesIO(zlib.decompress(data.read())) + nodecount = self.size["x"] * self.size["y"] * self.size["z"] + self.data = {} + for i in range(nodecount): + self.data[i] = Node(self.nodes[readU16(bulk)]) + + for i in range(nodecount): + self.data[i].set_param1(readU8(bulk)) + + for i in range(nodecount): + self.data[i].set_param2(readU8(bulk)) + + self.loaded = True + + def export(self): + if not self.loaded: + return + + data = BytesIO(b"") + + data.write(b"MTSM") + writeU16(data, self.version) + + writeU8(data, self.size["x"]) + writeU8(data, self.size["y"]) + writeU8(data, self.size["z"]) + + for u in range(self.size["y"]): + p = self.y_slice_probs.get(u) or 127 + writeU8(data, p) + + writeU16(data, len(self.nodes)) + for node in self.nodes: + writeU16(data, len(node)) + for c in node: + writeU8(data, ord(c)) + + bulk = BytesIO(b"") + nodecount = self.size["x"] * self.size["y"] * self.size["z"] + for i in range(nodecount): + writeU16(bulk, self.nodes.index(self.data[i].get_name())) + + for i in range(nodecount): + writeU8(bulk, self.data[i].get_param1()) + + for i in range(nodecount): + writeU8(bulk, self.data[i].get_param2()) + + bulk.seek(0) + data.write(zlib.compress(bulk.read())) + data.seek(0) + + return data + + def load_from_file(self, filename): + try: + ifile = open(filename, "rb") + except Exception as err: + print("ERROR: {0} couldn't open file {1} : {2}".format(self, filename, err)) + return + + self.load(ifile) + + def export_to_file(self, filename): + try: + ofile = open(filename, "wb") + except Exception as err: + print("ERROR: {0} couldn't open file {1} : {2}".format(self, filename, err)) + return + + ofile.write(self.export().read()) + + def serialize_schematic(self, schemtab): + self._init_data() + + self.version = 4 + self.size = schemtab["size"] + + if schemtab.get("y_slice_probs"): + for prob in schemtab["y_slice_probs"]: + self.y_slice_probs[prob[0]] = prob[1] + + for index in schemtab["data"]: + entry = schemtab["data"][index] + + if not entry["name"] in self.nodes: + self.nodes.append(entry["name"]) + + self.data[index] = Node(entry["name"], param1 = entry["prob"], param2 = entry.get("param2") or 0) + if not entry.get("force_place"): + self.data[index].set_param1(int(entry["prob"] / 2)) + + self.loaded = True + + def get_node(self, pos): + if not self.loaded: + return + + if pos.x > self.size["x"] or pos.y > self.size["y"] or pos.z > self.size["z"]: + return + + abspos = pos.x + (pos.y * self.size["x"]) + (pos.z * self.size["y"] * self.size["x"]) + return self.data[abspos] diff --git a/src/test.py b/src/test.py new file mode 100755 index 0000000..fbbab03 --- /dev/null +++ b/src/test.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3.4 +# -*- encoding: utf-8 -*- +############################ +## Tests ran for Python-MT +## + +import minetest +import random +from utils import readS8, readS16, Pos +from io import BytesIO +from schematics import Schematic + +def testSignedEndians(): + print(readS16(BytesIO(b"\x9f\xff"))) + +def testMapBlockLoad(): + file = minetest.map.MapVessel("./map.sqlite") + #i = 2+12*4096+(-9)*4096*4096 + #i = 0 + for i in range(-4096, 4096): + res, code = file.read(i) + if not res: + continue + else: + print("Read {0}: {1}".format(i, (res, code))) + + mapb = file.load(i) + print("Loaded {0}: {1}".format(i, mapb)) + + + print(len(file.cache)) + +def testGetNode(): + db = minetest.MapInterface("./map.sqlite") + for _ in range(1000): + pos = Pos({'x': random.randint(-300, 300), 'y': random.randint(-300, 300), 'z': random.randint(-300, 300)}) + + print("{0}: {1}".format(pos, db.get_node(pos).get_name())) + print("Cache size: {0}".format(len(db.interface.cache)), end = " \r") + assert(len(db.interface.cache) <= db.get_maxcachesize()) + +def testSetNode(): + db = minetest.MapInterface("./map.sqlite") + f = open("./dump.bin", "w") + dummy = minetest.Node("default:nyancat") + + for y in range(1, 10): + db.set_node(Pos({'x': 0, 'y': y, 'z': 0}), dummy) + + db.save() + +def invManip(): + db = minetest.MapInterface("./map.sqlite") + chest = db.get_meta(Pos({'x': 0, 'y': 0, 'z': 0})) + #print(chest) + inv = chest.get_inventory() + #print(inv) + #print(chest.get_string("formspec")) + #chest.set_string("formspec", chest.get_string("formspec") + "button[0,0;1,0.5;moo;Moo]") + #print(chest.get_string("formspec")) + print(inv.is_empty("main")) + print(inv.get_size("main")) + + db.save() + +def testSchematics(): + # Import from file + schem = Schematic("/home/lymkwi/.minetest/games/minetest_game/mods/default/schematics/apple_tree_from_sapling.mts") + + # Export to BytesStream & file + print(schem.export().read()) + schem.export_to_file("test.mts") + + # Map export + db = minetest.MapInterface("./map.sqlite") + schem = db.export_schematic(Pos(), Pos({"x": -100, "y": 10, "z": 100})) + schem.export_to_file("test.mts") + + # Get node + print(schem.get_node(Pos({"x": 1, "y": 1, "z": 0}))) + +def removeUnknowns(): + import sys + + if len(sys.argv) < 2: + print("I need a map.sqlite file!") + return + elif len(sys.argv) < 3: + print("I need a known nodes file!") + return + + try: + knodes = open(sys.argv[2]) + except Exception as err: + print("Couldn't open know nodes file {0} : {1}".format(sys.argv[1], err)) + return + + print("Know nodes file opened") + nodes = [node[:-1] for node in knodes.readlines()] # Remove the \n + print("{0} nodes known".format(len(nodes))) + + u = minetest.map.MapVessel(sys.argv[1]) + ma = 4096 + 4096 * 4096# + 4096 * 4096 * 4096 + for i in range(-ma, ma): + k = u.load(i) + absi = minetest.utils.posFromInt(i, 4096) + print("Testing mapblock {0} ({1}) ".format(i, absi), end = '\r') + if k: + print("Checking mapblock {0} ({1})".format(i, absi), end = " \r") + unknowns = [] + for id in k.name_id_mappings: + node = k.name_id_mappings[id] + if node != "air": + if not node in nodes: + print("Unknown node in {0} : {1}".format(i, node)) + unknowns.append(node) + + if len(unknowns) > 0: + for x in range(16): + for y in range(16): + for z in range(16): + noderef = k.get_node(x + y * 16 + z * 16 * 16) + if noderef.get_name() in unknowns: + print("Removed node in {0} : {1}".format(noderef.get_pos(), noderef.get_name())) + k.remove_node(x + y * 16 + z * 16 * 16) + + print("Saving mapblock {0}".format(absi)) + u.store(i, k.implode()) + u.write(i) + + u.uncache(i) + u.commit() + + + +if __name__ == "__main__": + #findTheShelves() + print("=> MapBlockLoad") + #testMapBlockLoad() + removeUnknowns() + """print("=> signed endians") + testSignedEndians() + print("=> get_node") + testGetNode() + print("=> set_node") + testSetNode() + print("=> inventory manipulation (WIP)") + invManip() + print("=> schematic manipulation (WIP)") + testSchematics()""" diff --git a/utils.py b/src/utils.py similarity index 99% rename from utils.py rename to src/utils.py index 0790ce7..6777d6d 100644 --- a/utils.py +++ b/src/utils.py @@ -138,4 +138,4 @@ def writeU32(strm, val): val -= k val /= 256 - strm.write(bytes(vals)) \ No newline at end of file + strm.write(bytes(vals)) diff --git a/test.py b/test.py deleted file mode 100755 index 4328818..0000000 --- a/test.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3.4 -# -*- encoding: utf-8 -*- -############################ -## Tests ran for Python-MT -## - -import minetest -import random -from utils import readS8, readS16, Pos -from io import BytesIO - - -def findTheShelves(): - from PIL import Image - file = minetest.MapInterface("/home/lymkwi/.minetest/worlds/NodesJustWannaHaveFun/map.sqlite") - file.setMaxCacheSize(450) - mapy = {} - size = 16 # In mapblocks - hsize = int(size/2) - alphas, noairs = {}, {} - for x in range(-hsize, hsize): - for z in range(-hsize, hsize): - for y in range(0, 16): - posx, posy, posz = x*16, y*16, z*16 - - for intx in range(0, 16): - for intz in range(0, 16): - alpha, noair = 0, 0 - for inty in range(0, 16): - node = file.get_node(Pos({'x': posx+intx, 'y': posy+inty, 'z': posz+intz})) - print("[{0}] {1}".format(node.pos, node.get_name()), end = (' ' * 20) + '\r') - if node.get_name() != "air": - noair += 1 - if node.get_name() == "ignore": - alpha += 1 - - coords = ((posx + intx) * 4096 * (posz + intz)) - alphas[coords] = (alphas.get(coords) or 256) - alpha - noairs[coords] = (noairs.get(coords) or 0) + noair - -# mapy = [(alpha,) * 3 + (noair,) ] - for alpha, noair in zip(alphas.keys(), noairs.keys()): - mapy[alpha] = (alphas[alpha],) * 3 + (noairs[noair],) - - buildPic(mapy, size).save("map.jpg") - - -def buildPic(mapy, size): - im = Image.new("RGBA", (size*16, size*16)) - hsize = int(size*8) - for x in range(-hsize, hsize): - for z in range(-hsize, hsize): - im.putpixel((z+hsize, x+hsize), mapy[x * z*4096]) - - im.show() - return im - -def testSignedEndians(): - print(readS16(BytesIO(b"\x9f\xff"))) - -def testMapBlockLoad(): - file = minetest.map.MapVessel("./map.sqlite") - #i = 2+12*4096+(-9)*4096*4096 - #i = 0 - for i in range(-4096, 4096): - res, code = file.read(i) - if not res: - continue - else: - print("Read {0}: {1}".format(i, (res, code))) - - mapb = file.load(i) - print("Loaded {0}: {1}".format(i, mapb)) - - - print(len(file.cache)) - -def testGetNode(): - db = minetest.MapInterface("./map.sqlite") - for _ in range(1000): - pos = Pos({'x': random.randint(-300, 300), 'y': random.randint(-300, 300), 'z': random.randint(-300, 300)}) - - print("{0}: {1}".format(pos, db.get_node(pos).get_name())) - print("Cache: {0}".format(len(db.interface.cache))) - -def testSetNode(): - db = minetest.MapInterface("./map.sqlite") - f = open("./dump.bin", "w") - dummy = minetest.Node("default:nyancat") - - for y in range(1, 256): - db.set_node(Pos({'x': 0, 'y': y, 'z': 0}), dummy) - - db.save() - -def invManip(): - db = minetest.MapInterface("./map.sqlite") - chest = db.get_meta(Pos({'x': 0, 'y': 0, 'z': 0})) - #print(chest) - inv = chest.get_inventory() - #print(inv) - #print(chest.get_string("formspec")) - #chest.set_string("formspec", chest.get_string("formspec") + "button[0,0;1,0.5;moo;Moo]") - #print(chest.get_string("formspec")) - print(inv.is_empty("main")) - print(inv.get_size("main")) - - db.save() - -if __name__ == "__main__": - #findTheShelves() - #testMapBlockLoad() - #testSignedEndians() - #testGetNode() - #testSetNode() - invManip() diff --git a/tools/python-minetest/dumpnodes.lua b/tools/python-minetest/dumpnodes.lua new file mode 100644 index 0000000..9450c63 --- /dev/null +++ b/tools/python-minetest/dumpnodes.lua @@ -0,0 +1,16 @@ +minetest.register_chatcommand("dumpnodes", { + privs = {server = true}, + description = "Dumps all known nodes in a file", + func = function() + local f = io.open(minetest.get_modpath("devel") .. "/knownnodes.txt", "w") + for node in pairs(minetest.registered_nodes) do + f:write(node) + f:write('\n') + end + + f:flush() + f:close() + + return true, "Nodes dumped in " .. minetest.get_modpath("devel") .. "/knownnodes.txt" + end +}) diff --git a/tools/python-minetest/init.lua b/tools/python-minetest/init.lua new file mode 100644 index 0000000..7664c21 --- /dev/null +++ b/tools/python-minetest/init.lua @@ -0,0 +1,5 @@ +-- Mod to use for python-minetest +-- License : WTFPL +-- By Mg/LeMagnesium + +dofile(minetest.get_modpath("devel") .. "/dumpnodes.lua")