From 4f103d19fd3f6b0c5f4f5cee2ea6cb40208fdadb Mon Sep 17 00:00:00 2001 From: LeMagnesium Date: Sun, 24 Jan 2016 17:08:28 +0100 Subject: [PATCH] Initial commit --- .gitignore | 2 + errors.py | 43 +++++++ inventory.py | 88 +++++++++++++ map.py | 341 +++++++++++++++++++++++++++++++++++++++++++++++++++ metadata.py | 50 ++++++++ minetest.py | 10 ++ nodes.py | 35 ++++++ test.py | 34 +++++ utils.py | 96 +++++++++++++++ 9 files changed, 699 insertions(+) create mode 100644 .gitignore create mode 100644 errors.py create mode 100644 inventory.py create mode 100644 map.py create mode 100644 metadata.py create mode 100644 minetest.py create mode 100644 nodes.py create mode 100755 test.py create mode 100644 utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..97d90e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +map.sqlite diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..d5c645b --- /dev/null +++ b/errors.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Rrrors for Python-MT +## +## +# + + +class MinetestException(Exception): + pass + +##=========================================## +# 1. Map errors + +class MapError(MinetestException): + pass + +class EmptyMapVesselError(MinetestException): + __cause__ = "Tried to use empty mapfile vessel" + +class UnknownMetadataTypeIDError(MapError): + pass + +class InvalidParamLengthError(MapError): + pass + +##=========================================## +# 2. Containers Errors +class ContainerError(MinetestException): + __cause__ = "Error in container" + +class OutOfBordersCoordinates(ContainerError): + __cause__ = "Coordinates out of borders" + +##=========================================## +# 3. Inventory Errors + +class InventoryError(MinetestException): + __cause__ = "Inventory Error" + +class InventoryDeserializationError(InventoryError): + pass diff --git a/inventory.py b/inventory.py new file mode 100644 index 0000000..72eb2f0 --- /dev/null +++ b/inventory.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Inventory for Python-MT +## +## +# + +from io import StringIO + +from errors import InventoryDeserializationError +from utils import readU8 + +def getSerializedInventory(strm): + # serialized inventory + inv = "".join([chr(readU8(strm)) for _ in range(len(b"EndInventory\n"))]) + while not "EndInventory\n" in inv: + inv += chr(readU8(strm)) + + return inv + + +def deserializeInventory(serializat): + newlists = {} + lines = serializat.split('\n') + expectation = 0 + current_listname = "" + expected_type = None + + for line in lines: + params = line.split(' ') + + if params[0] == "": + # EOF + break + + elif params[0] == "List": + newlists[params[1]] = [] + expectation = int(params[2]) + expected_type = "ItemStack" + current_listname = params[1] + + elif params[0] == "Empty" or params[0] == "Item": + if expectation > 0: + expectation -= 1 + else: + raise InventoryDeserializationError("Unexpected Item (line {0})".format(lines.index(line)+1)) + + newlists[current_listname].append(ItemStack("".join(params[1:] or []))) + + elif params[0] == "EndInventoryList": + if expectation > 0: + raise InventoryDeserializationError("Too few items for list {0}".format(current_listname)) + + expected_type = "ItemStack" + current_listname = "" + + return newlists + + + +class ItemStack: + def __init__(self, deft): + if not deft: + self.name = "" + self.count = 0 + + elif type(deft) == type("str"): + self.deserialize(deft) + + elif type(deft) == type({}): + self.name = deft["name"] + self.count = deft["count"] + + def deserialize(self, serializat): + pass + + + +class InvRef: + def __init__(self, fields = {}): + self.lists = fields + + def from_string(self, serializat): + self.lists = deserializeInventory(serializat) + + def from_list(self, lists): + self.lists = lists diff --git a/map.py b/map.py new file mode 100644 index 0000000..0d4a44a --- /dev/null +++ b/map.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Maps for Python-MT +## +## +# + +import sqlite3 as _sql +import zlib +from io import BytesIO + +from errors import MapError, EmptyMapVesselError, UnknownMetadataTypeIDError, InvalidParamLengthError +from utils import readU8, readU16, readU32, readS32, Pos, posFromInt +from metadata import NodeMetaRef +from inventory import getSerializedInventory +from nodes import NodeTimerRef + +# Bitmask constants +IS_UNDERGROUND = 1 +DAY_NIGHT_DIFFERS = 2 +LIGHTING_EXPIRED = 4 +GENERATED = 8 + +class MapBlock: + def __init__(self, data = None): + if data: + self.explode(data) + else: + self.version = 0 + self.bitmask = b"08" + self.content_width = 2 + self.param_width = 2 + self.node_data = dict() + self.node_meta = dict() + self.node_timers = dict() + self.static_object_version = 0 #u8 + self.static_object_count = 0 #u16 + self.static_objects = [] #u8, s32, s32, s32, u16, u8 + self.timestamp = 0 #u32 + self.name_id_mapping_version = 0 #u8 + self.num_name_id_mappings = 0 #u16 + self.name_id_mappings = dict() #u16, u8[u16] + self.single_timer_data_length = 10 #u8 + self.timer_counts = 0 #u16 + self.timers = dict() #u16, s32, s32 + + def explode(self, bytelist): + data = BytesIO(bytelist) + + self.mapblocksize = 16 # Normally + self.version = readU8(data) + self.bitmask = readU8(data) + self.content_width = readU8(data) + self.param_width = readU8(data) + + self.node_data = dict() + + k = b"" + while True: + oldklen = len(k) + k += data.read(1) + + try: + c_width_data = BytesIO(zlib.decompress(k)) + except zlib.error as err: + if len(k) > oldklen: + continue + else: + break + + self.node_data["param0"] = [ int(b) for b in c_width_data.read(4096 * self.content_width) ] + self.node_data["param1"] = [ int(b) for b in c_width_data.read(4096) ] + self.node_data["param2"] = [ int(b) for b in c_width_data.read(4096) ] + + try: + assert(len(self.node_data["param0"]) == 4096 * self.content_width) + assert(len(self.node_data["param1"]) == 4096) + assert(len(self.node_data["param2"]) == 4096) + except AssertError: + raise InvalidParamLengthError() + + k = b"" + while True: + oldklen = len(k) + k += data.read(1) + + try: + node_meta_list = BytesIO(zlib.decompress(k)) + except zlib.error as err: + if len(k) > oldklen: + continue + else: + break + + self.node_meta = dict() + if self.version <= 22: + self.meta_version = readU16(node_meta_list) + self.metadata_count = readU16(node_meta_list) + + for i in range(self.metadata_count): + pos = posFromInt(readU16(node_meta_list), self.mapblocksize).getAsTuple() + self.node_meta[pos] = NodeMetaRef(pos) + + type_id = readU16(node_meta_list) + c_size = readU16(node_meta_list) + meta = [readU8(node_meta_list) for _ in range(c_size)] + + if type_id == 1: + # It is "generic" metadata + + # serialized inventory + self.node_meta[pos].get_inventory().from_list(getSerializedInventory(node_meta_list)) + + # u8[u32 len] text + self.node_meta[pos].set_raw("text", "".join([ readU8(node_meta_list) for _ in range(readU32(node_meta_list))])) + + # u8[u16 len] owner + self.node_meta[pos].set_raw("owner", "".join([ readU8(node_meta_list) for _ in range(readU16(node_meta_list))])) + + # u8[u16 len] infotext + self.node_meta[pos].set_raw("infotext", "".join([ readU8(node_meta_list) for _ in range(readU16(node_meta_list))])) + + # u8[u16 len] inventory_drawspec + self.node_meta[pos].set_raw("formspec", "".join([ readU8(node_meta_list) for _ in range(readU16(node_meta_list))])) + + # u8 allow_text_input + self.node_meta[pos].set_raw("allow_text_input", readU8(node_meta_list)) + + # u8 removeal_disabled + self.node_meta[pos].set_raw("removal_disabled", readU8(node_meta_list)) + + # u8 enforce_owner + self.node_meta[pos].set_raw("enforce_owner", readU8(node_meta_list)) + + # u32 num_vars + num_vars = readU32(node_meta_list) + + for _ in range(num_vars): + # u8 [u16 len] name + name = [readU8(node_meta_list) for _ in range(readU16(node_meta_list))] + + # u8 [u32 len] value + value = [readU8(node_meta_list) for _ in range(readU32(node_meta_list))] + + self.node_meta[pos].set_raw(name, value) + + elif type_id == 14: + # Sign metadata + # u8 [u16 text_len] text + self.node_meta[pos].set_raw("text", "".join([ readU8(node_meta_list) for _ in range(readU16(node_meta_list)) ])) + + elif type_id == 15 or type_id == 16: + # Chest metadata + # Also, Furnace metadata + # Which doesn't seem to be documented + # So let's assume they're like chests + # (which will probably fail) + + # serialized inventory + self.node_meta[pos].get_inventory().from_list(getSerializedInventory(node_meta_list)) + + + elif type_id == 17: + # Locked Chest metadata + + # u8 [u16 len] owner + self.node_meta[pos].set_raw("owner", "".join([ readU8(node_meta_list) for _ in range(readU16(node_meta_list)) ])) + + # serialized inventory + self.node_meta[pos].get_inventory().from_list(getSerializedInventory(node_meta_list)) + + else: + raise UnknownMetadataTypeIDError("Unknown metadata type ID: {0}".format(type_id)) + + else: + self.meta_version = readU8(node_meta_list) + if self.meta_version == 0:# and self.bitmask & GENERATED == 0: + # Mapblock was probably not generated + # It is CONTENT_IGNORE + # Or there are no metadata + # GET THE HELL OUT OF HERE! + pass + + else: + self.metadata_count = readU16(node_meta_list) + + for _ in range(self.metadata_count): + pos = posFromInt(readU16(node_meta_list), self.mapblocksize).getAsTuple() + self.node_meta[pos] = NodeMetaRef(pos) + + num_vars = readU32(node_meta_list) + for _ in range(num_vars): + key_len = readU16(node_meta_list) + key = "".join([chr(readU8(node_meta_list)) for _ in range(key_len)]) + + val_len = readU32(node_meta_list) + val = [readU8(node_meta_list) for _ in range(val_len)] + self.node_meta[pos].set_raw(key, val) + + self.node_meta[pos].get_inventory().from_list(getSerializedInventory(node_meta_list)) + + # We skip node_timers for now, not used in v23, v24 never released, and v25 has them later + + # u8 static_object_version + self.static_object_version = readU8(data) + + # u16 static_object_count + self.static_object_count = readU16(data) + + self.static_objects = [] + for _ in range(self.static_object_count): + # u8 type + otype = readU8(data) + + # s32 pos_x_nodes + pos_x_nodes = readS32(data) / 10000 + + # s32 pos_y_nodes + pos_y_nodes = readS32(data) / 10000 + + # s32 pos_z_nodes + pos_z_nodes = readS32(data) / 10000 + + + # u8 [u16 data_size] data + odata = [ readU8(data) for _ in range(readU16(data)) ] + + self.static_objects.append({ + "type": otype, + "pos": Pos({'x': pos_x_nodes, 'y': pos_y_nodes, 'z': pos_z_nodes}), + "data": odata, + }) + + # u32 timestamp + self.timestamp = readU32(data) + + # u8 name_id_mapping_version + self.name_id_mapping_version = readU8(data) + + # u16 num_name_id_mappings + self.num_name_id_mappings = readU16(data) + + self.name_id_mappings = dict() + for _ in range(self.num_name_id_mappings): + # u16 id, u8 [u16 name_len] name + id = readU16(data) + name = [ readU8(data) for _ in range(readU16(data)) ] + self.name_id_mappings[id] = name + + if self.version == 25: + # u8 single_timer_data_length + self.single_timer_data_length = readU8(data) + + # u16 num_of_timers + self.timer_counts = readU16(data) + + self.timers = dict() + for _ in range(self.timer_counts): + pos = posFromInt(readU16(data), 16).getAsTuple() + timeout = readS32(data) / 1000 + elapsed = readS32(data) / 1000 + self.timers[pos] = NodeTimerRef(pos, timeout, elapsed) + + # EOF! + +class MapVessel: + def __init__(self, mapfile, backend = "sqlite3"): + self.mapfile = mapfile + self.cache = dict() + self.open(mapfile, backend) + + def __str__(self): + if self.isEmpty(): + return "empty mapfile vessel" + else: + return "mapfile vessel for {0}".format(self.mapfile) + + def isEmpty(self): + return self.mapfile == None + + def open(self, mapfile, backend = "sqlite3"): + try: + self.conn = _sql.connect(mapfile) + self.cur = self.conn.cursor() + except _sql.OperationalError as err: + raise MapError("Error opening database : {0}".format(err)) + + def close(self): + self.conn.close() + self.cache = None + self.mapfile = None + + def read(self, blockID): + if self.isEmpty(): + raise EmptyMapVesselError() + + if self.cache.get(blockID): + return False, "dejavu" + + try: + self.cur.execute("SELECT * from blocks where pos = {0}".format(blockID)) + except _sql.OperationalError as err: + raise MapError(err) + + data = self.cur.fetchall() + if len(data) == 1: + self.cache[blockID] = data[0][1] + return True, "ok" + else: + return False, "notfound" + + def uncache(self, blockID): + if self.isEmpty(): + raise EmptyMapVesselError() + + self.cache[blockID] = None + del self.cache[blockID] + return True, "ok" + + def write(self, blockID): + if self.isEmpty(): + raise EmptyMapVesselError() + + if not self.cache.get(blockID): + return False, "notread" + + try: + self.cur.execute("REPLACE INTO 'blocks' ('pos', 'data') VALUES ({0}, ?)".format(blockID), + [self.cache[blockID]]) + except _sql.OperationalError as err: + raise MapError(err) + + def load(self, blockID): + if self.isEmpty(): + raise EmptyMapVesselError() + + if not self.cache.get(blockID): + return False, "notread" + + return MapBlock(self.cache[blockID]) diff --git a/metadata.py b/metadata.py new file mode 100644 index 0000000..96126e2 --- /dev/null +++ b/metadata.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Metadata for Python-MT +## +## +# + +from inventory import InvRef +from utils import Pos + + +class NodeMetaRef: + def __init__(self, spos = Pos(), meta = dict(), inv = InvRef()): + self.data = meta + self.pos = spos + self.inv = inv + + def get_raw(self, key): + return self.data.get(key) + + def set_raw(self, key, val): + self.data[key] = val + + def get_string(self, key): + return str(self.data.get(key)) + + def set_string(self, key, val): + self.data[key] = str(val) + + def get_int(self, key): + return int(self.data.get(key)) + + def set_int(self, key, val): + self.data[key] = int(val) + + def get_float(self, key): + return float(self.data.get(key)) + + def set_float(self, key, val): + self.data[key] = float(val) + + def get_inventory(self): + return self.inv + + def to_table(self): + return self.meta + + def from_table(self, tab = {}): + self.meta = tab diff --git a/minetest.py b/minetest.py new file mode 100644 index 0000000..031dd99 --- /dev/null +++ b/minetest.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +################################## +## Python Library to manipulate +## Minetest's files +## + +import utils +from utils import Pos +from map import MapVessel diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..6ad3e5d --- /dev/null +++ b/nodes.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Maps for Python-MT +## +## +# + +from utils import Pos + +class NodeTimerRef: + def __init__(self, pos = Pos(), timeout = 0.0, elapsed = 0.0): + self.pos = pos + self.timeout = timeout + self.elapsed = elapsed + self.active = False + + def set(self, timeout, elapsed): + self.timeout, self.elapsed = timeout, elapsed + + def start(self, timeout): + self.set(timeout, 0) + self.active = True + + def stop(self): + self.active = False + + def get_timeout(self): + return self.timeout + + def get_elapsed(self): + return self.elapsed + + def is_started(self): + return self.active diff --git a/test.py b/test.py new file mode 100755 index 0000000..6d538df --- /dev/null +++ b/test.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +########################## +## + +import minetest +import random +from utils import readS8, readS16 +from io import BytesIO + +file = minetest.MapVessel("./map.sqlite") +#i = 2+12*4096+(-9)*4096*4096 +#i = 0 + +def testSignedEndians(): + print(readS16(BytesIO(b"\x9f\xff"))) + +def testMapBlockLoad(): + for i in range(-4096, 4096): + res, code = file.read(i) + if not res: + continue + else: + print("Read {0}: {1}".format(i, (res, code))) + + mapb = file.load(i) + print("Loaded {0}: {1}".format(i, mapb)) + + + print(len(file.cache)) + +if __name__ == "__main__": + testMapBlockLoad() + #testSignedEndians() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..9b7e9f5 --- /dev/null +++ b/utils.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- encoding: utf8 -*- +########################### +## Utils for Python-MT +## +## +# + +from io import BytesIO + +def posFromInt(pos, blocksize): + posx, posy = 0, 0 + + posx = pos%blocksize + pos -= posx + pos = int(pos/blocksize) + + posy = pos%blocksize + pos -= posy + pos = int(pos/blocksize) + + return Pos({'x': posx, 'y': posy, 'z': pos}) + +def int64(u): + while u >= 2**63: + u -= 2**64 + while u <= -2**63: + u += 2**64 + return u + +def getIntegerAsBlock(i): + x = unsignedToSigned(i % 4096, 2048) + i = int((i - x) / 4096) + y = unsignedToSigned(i % 4096, 2048) + i = int((i - y) / 4096) + z = unsignedToSigned(i % 4096, 2048) + i = int((i - z) / 4096) + return Pos({"x": x, "y": y, "z": z}) + +def unsignedToSigned(i, max_positive): + if i < max_positive: + return i + else: + return i - 2*max_positive + +class Pos: + def __init__(self, posdict = {"x": 0, "y": 0, "z": 0}): + self.dict = posdict + self.x = posdict.get("x") or 0 + self.y = posdict.get("y") or 0 + self.z = posdict.get("z") or 0 + + def __str__(self): + return "({0}, {1}, {2})".format(self.x, self.y, self.z) + + def __eq__(self, other): + return self.x == other.x and self.y == other.y and self.z == other.z + + def getAsInt(self): + return int64(self.z * 4096 * 4096 + self.y * 4096 + self.x) + + def getAsTuple(self): + return (self.x, self.y, self.z) + +# Big-endian!!! +def readU8(strm): + return (ord(strm.read(1))) + +def readU16(strm): + #return (ord(strm.read(1)) << 16) + (ord(strm.read(1))) + return (ord(strm.read(1)) << 8) + (ord(strm.read(1))) + +def readU32(strm): + return (ord(strm.read(1)) << 24) + (ord(strm.read(1)) << 16) + (ord(strm.read(1)) << 8) + (ord(strm.read(1))) + +# Works with eight-bit two's complement +def readS8(strm): + u = readU8(strm) + if u & pow(2,7): # Negative + return -pow(2,7) + (u-pow(2,7)) + else: + return u + +def readS16(strm): + u = readU16(strm) + if u & pow(2,15): # Negative + return -pow(2,15) + (u-pow(2,15)) + else: + return u + +def readS32(strm): + u = readU32(strm) + if u & pow(2,31): # Negative + return -pow(2,31) + (u-pow(2,31)) + else: + return u