Optimize database access - load only blocks that are needed

TileGenerator.cpp

@@ -95,6 +95,7 @@ static inline int colorSafeBounds(int color)
 
 TileGenerator::TileGenerator():
 	verboseCoordinates(false),
+	verboseStatistics(false),
 	m_bgColor(255, 255, 255),
 	m_scaleColor(0, 0, 0),
 	m_originColor(255, 0, 0),
@@ -408,7 +409,7 @@ void TileGenerator::loadBlocks()
 		if (pos.z > m_zMax) {
 			m_zMax = pos.z;
 		}
-		m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
+		m_positions.push_back(pos);
 	}
 	if (verboseCoordinates) {
 		cout << "World Geometry:       "
@@ -445,7 +446,6 @@ void TileGenerator::loadBlocks()
 			<< ")    blocks: " << map_blocks << "\n";
 	}
 	m_positions.sort();
-	m_positions.unique();
 }
 
 inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
@@ -484,123 +484,159 @@ std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
 	return out;
 }
 
+TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
+{
+	int64_t iPos;
+	iPos =  pos.x;
+	iPos += static_cast<int64_t>(pos.y) << 12;
+	iPos += static_cast<int64_t>(pos.z) << 24;
+	DBBlock in = m_db->getBlockOnPos(iPos);
+	Block out(pos,(const unsigned char *)"");
+
+	if (!in.second.empty()) {
+		out = Block(decodeBlockPos(in.first), in.second);
+		// Verify. Just to be sure...
+		if (pos.x != out.first.x || pos.y != out.first.y || pos.z != out.first.z) {
+			std::ostringstream oss;
+			oss << "Got unexpexted block: "
+			    << out.first.x << "," << out.first.y << "," << out.first.z
+			    << " from database. Requested: "
+			    << pos.x << "," << pos.y << "," << pos.z;
+			throw std::runtime_error(oss.str());
+		}
+	}
+	else {
+		// I can't imagine this to be possible...
+		std::ostringstream oss;
+		oss << "Failed to get block: "
+		    << pos.x << "," << pos.y << "," << pos.z
+		    << " from database.";
+		throw std::runtime_error(oss.str());
+	}
+	return out;
+}
+
 void TileGenerator::renderMap()
 {
-	std::list<int> zlist = getZValueList();
+	int blocks_selected = 0;
-	for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
+	int blocks_rendered = 0;
-		int zPos = *zPosition;
+	BlockPos currentPos;
-		std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
+	currentPos.x = INT_MIN;
-		for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
+	currentPos.y = 0;
-			if (position->second != zPos) {
+	currentPos.z = INT_MIN;
-				continue;
+	bool allReaded = false;
-			}
+	for (std::list<BlockPos>::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
-
+		const BlockPos &pos = *position;
+		if (currentPos.x != pos.x || currentPos.z != pos.z) {
+			if (currentPos.z != pos.z && currentPos.z != INT_MIN && m_shading)
+				renderShading(currentPos.z);
 			for (int i = 0; i < 16; ++i) {
 				m_readedPixels[i] = 0;
 			}
+			currentPos = pos;
+		}
+		else if (allReaded) {
+			continue;
+		}
+		Block block = getBlockOnPos(pos);
+		blocks_selected++;
+		if (!block.second.empty()) {
+			const unsigned char *data = block.second.c_str();
+			size_t length = block.second.length();
 
-			int xPos = position->first;
+			uint8_t version = data[0];
-			blocks[xPos].sort();
+			//uint8_t flags = data[1];
-			const BlockList &blockStack = blocks[xPos];
-			for (BlockList::const_iterator it = blockStack.begin(); it != blockStack.end(); ++it) {
-				const BlockPos &pos = it->first;
-				const unsigned char *data = it->second.c_str();
-				size_t length = it->second.length();
 
-				uint8_t version = data[0];
+			size_t dataOffset = 0;
-				//uint8_t flags = data[1];
+			if (version >= 22) {
+				dataOffset = 4;
+			}
+			else {
+				dataOffset = 2;
+			}
 
-				size_t dataOffset = 0;
+			ZlibDecompressor decompressor(data, length);
-				if (version >= 22) {
+			decompressor.setSeekPos(dataOffset);
-					dataOffset = 4;
+			ZlibDecompressor::string mapData = decompressor.decompress();
-				}
+			ZlibDecompressor::string mapMetadata = decompressor.decompress();
-				else {
+			dataOffset = decompressor.seekPos();
-					dataOffset = 2;
-				}
 
-				ZlibDecompressor decompressor(data, length);
+			// Skip unused data
-				decompressor.setSeekPos(dataOffset);
+			if (version <= 21) {
-				ZlibDecompressor::string mapData = decompressor.decompress();
-				ZlibDecompressor::string mapMetadata = decompressor.decompress();
-				dataOffset = decompressor.seekPos();
-
-				// Skip unused data
-				if (version <= 21) {
-					dataOffset += 2;
-				}
-				if (version == 23) {
-					dataOffset += 1;
-				}
-				if (version == 24) {
-					uint8_t ver = data[dataOffset++];
-					if (ver == 1) {
-						uint16_t num = readU16(data + dataOffset);
-						dataOffset += 2;
-						dataOffset += 10 * num;
-					}
-				}
-
-				// Skip unused static objects
-				dataOffset++; // Skip static object version
-				int staticObjectCount = readU16(data + dataOffset);
 				dataOffset += 2;
-				for (int i = 0; i < staticObjectCount; ++i) {
+			}
-					dataOffset += 13;
+			if (version == 23) {
-					uint16_t dataSize = readU16(data + dataOffset);
+				dataOffset += 1;
-					dataOffset += dataSize + 2;
+			}
-				}
+			if (version == 24) {
-				dataOffset += 4; // Skip timestamp
+				uint8_t ver = data[dataOffset++];
-
+				if (ver == 1) {
-				m_blockAirId = -1;
+					uint16_t num = readU16(data + dataOffset);
-				m_blockIgnoreId = -1;
-				// Read mapping
-				if (version >= 22) {
-					dataOffset++; // mapping version
-					uint16_t numMappings = readU16(data + dataOffset);
 					dataOffset += 2;
-					for (int i = 0; i < numMappings; ++i) {
+					dataOffset += 10 * num;
-						uint16_t nodeId = readU16(data + dataOffset);
-						dataOffset += 2;
-						uint16_t nameLen = readU16(data + dataOffset);
-						dataOffset += 2;
-						string name = string(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
-						if (name == "air") {
-							m_blockAirId = nodeId;
-						}
-						else if (name == "ignore") {
-							m_blockIgnoreId = nodeId;
-						}
-						else {
-							m_nameMap[nodeId] = name;
-						}
-						dataOffset += nameLen;
-					}
 				}
+			}
 
-				// Node timers
+			// Skip unused static objects
-				if (version >= 25) {
+			dataOffset++; // Skip static object version
-					dataOffset++;
+			int staticObjectCount = readU16(data + dataOffset);
-					uint16_t numTimers = readU16(data + dataOffset);
+			dataOffset += 2;
+			for (int i = 0; i < staticObjectCount; ++i) {
+				dataOffset += 13;
+				uint16_t dataSize = readU16(data + dataOffset);
+				dataOffset += dataSize + 2;
+			}
+			dataOffset += 4; // Skip timestamp
+
+			m_blockAirId = -1;
+			m_blockIgnoreId = -1;
+			// Read mapping
+			if (version >= 22) {
+				dataOffset++; // mapping version
+				uint16_t numMappings = readU16(data + dataOffset);
+				dataOffset += 2;
+				for (int i = 0; i < numMappings; ++i) {
+					uint16_t nodeId = readU16(data + dataOffset);
 					dataOffset += 2;
-					dataOffset += numTimers * 10;
+					uint16_t nameLen = readU16(data + dataOffset);
-				}
+					dataOffset += 2;
-
+					string name = string(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
-				renderMapBlock(mapData, pos, version);
+					if (name == "air") {
-
+						m_blockAirId = nodeId;
-				bool allReaded = true;
-				for (int i = 0; i < 16; ++i) {
-					if (m_readedPixels[i] != 0xffff) {
-						allReaded = false;
 					}
+					else if (name == "ignore") {
+						m_blockIgnoreId = nodeId;
+					}
+					else {
+						m_nameMap[nodeId] = name;
+					}
+					dataOffset += nameLen;
 				}
-				if (allReaded) {
+			}
-					break;
+
+			// Node timers
+			if (version >= 25) {
+				dataOffset++;
+				uint16_t numTimers = readU16(data + dataOffset);
+				dataOffset += 2;
+				dataOffset += numTimers * 10;
+			}
+
+			renderMapBlock(mapData, pos, version);
+			blocks_rendered++;
+
+			allReaded = true;
+			for (int i = 0; i < 16; ++i) {
+				if (m_readedPixels[i] != 0xffff) {
+					allReaded = false;
 				}
 			}
 		}
-		if(m_shading)
-			renderShading(zPos);
 	}
+	if(currentPos.z != INT_MIN && m_shading)
+		renderShading(currentPos.z);
+	if (verboseStatistics)
+		cout << "Statistics:  Blocks selected: " << blocks_selected << ";  blocks rendered: " << blocks_rendered << std::endl;
 }
 
 inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version)
@@ -729,8 +765,8 @@ void TileGenerator::renderPlayers(const std::string &inputPath)
 inline std::list<int> TileGenerator::getZValueList() const
 {
 	std::list<int> zlist;
-	for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
+	for (std::list<BlockPos>::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
-		zlist.push_back(position->second);
+		zlist.push_back(position->z);
 	}
 	zlist.sort();
 	zlist.unique();

TileGenerator.h

@@ -32,6 +32,11 @@ struct BlockPos {
 	int x;
 	int y;
 	int z;
+	// operator< should order the positions in the
+	// order the corresponding pixels are generated:
+	// First (most significant): z coordinate, descending (i.e. reversed)
+	// Then                    : x coordinate, ascending
+	// Last (least significant): y coordinate, descending (i.e. reversed)
 	bool operator<(const BlockPos& p) const
 	{
 		if (z > p.z) {
@@ -40,19 +45,32 @@ struct BlockPos {
 		if (z < p.z) {
 			return false;
 		}
+		if (x < p.x) {
+			return true;
+		}
+		if (x > p.x) {
+			return false;
+		}
 		if (y > p.y) {
 			return true;
 		}
 		if (y < p.y) {
 			return false;
 		}
-		if (x > p.x) {
+		return false;
-			return true;
+	}
-		}
+	bool operator==(const BlockPos& p) const
-		if (x < p.x) {
+	{
+		if (z != p.z) {
 			return false;
 		}
-		return false;
+		if (y != p.y) {
+			return false;
+		}
+		if (x != p.x) {
+			return false;
+		}
+		return true;
 	}
 };
 
@@ -92,6 +110,7 @@ private:
 	void createImage();
 	void renderMap();
 	std::list<int> getZValueList() const;
+	Block getBlockOnPos(BlockPos pos);
 	std::map<int, BlockList> getBlocksOnZ(int zPos);
 	void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
 	void renderShading(int zPos);
@@ -105,6 +124,7 @@ private:
 
 public:
 	bool verboseCoordinates;
+	bool verboseStatistics;
 
 private:
 	Color m_bgColor;
@@ -138,7 +158,7 @@ private:
 	int m_reqYMaxNode;	// Node offset within a map block
 	int m_mapWidth;
 	int m_mapHeight;
-	std::list<std::pair<int, int> > m_positions;
+	std::list<BlockPos> m_positions;
 	std::map<int, std::string> m_nameMap;
 	ColorMap m_colors;
 	uint16_t m_readedPixels[16];

db-leveldb.cpp

@@ -62,3 +62,16 @@ DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
 	return blocks;
 }
 
+DBBlock DBLevelDB::getBlocksOnPos(int64_t iPos)
+{
+	DBBlock block(0,(const unsigned char *)"");
+	std::string datastr;
+	leveldb::Status status;
+
+	status = m_db->Get(leveldb::ReadOptions(), i64tos(Pos), &datastr);
+	if(status.ok())
+		block = DBBlock( iPos, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) );
+
+	return block;
+}
+

db-leveldb.h

@@ -10,6 +10,7 @@ public:
 	DBLevelDB(const std::string &mapdir);
 	virtual std::vector<int64_t> getBlockPos();
 	virtual DBBlockList getBlocksOnZ(int zPos);
+	virtual DBBlock getBlockOnPos(int64_t iPos);
 	~DBLevelDB();
 private:
 	leveldb::DB *m_db;

db-sqlite3.cpp

@@ -77,3 +77,34 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
 	return blocks;
 }
 
+DBBlock DBSQLite3::getBlockOnPos(int64_t iPos)
+{
+	std::string sql = "SELECT pos, data FROM blocks WHERE pos == ?";
+	if (!m_getBlocksOnPosStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_getBlocksOnPosStatement, 0) != SQLITE_OK) {
+		throw std::runtime_error("Failed to prepare statement");
+	}
+	DBBlock block(0,(const unsigned char *)"");
+
+	sqlite3_int64 psPos = static_cast<sqlite3_int64>(iPos);
+	sqlite3_bind_int64(m_getBlocksOnPosStatement, 1, psPos);
+
+	int result = 0;
+	while (true) {
+		result = sqlite3_step(m_getBlocksOnPosStatement);
+		if(result == SQLITE_ROW) {
+			sqlite3_int64 blocknum = sqlite3_column_int64(m_getBlocksOnPosStatement, 0);
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_getBlocksOnPosStatement, 1));
+			int size = sqlite3_column_bytes(m_getBlocksOnPosStatement, 1);
+			block = DBBlock(blocknum, std::basic_string<unsigned char>(data, size));
+			break;
+		} else if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else {
+			break;
+		}
+	}
+	sqlite3_reset(m_getBlocksOnPosStatement);
+
+	return block;
+}
+

db-sqlite3.h

@@ -3,12 +3,14 @@
 
 #include "db.h"
 #include <sqlite3.h>
+#include <map>
 
 class DBSQLite3 : public DB {
 public:
 	DBSQLite3(const std::string &mapdir);
 	virtual std::vector<int64_t> getBlockPos();
 	virtual DBBlockList getBlocksOnZ(int zPos);
+	virtual DBBlock getBlockOnPos(int64_t iPos);
 	~DBSQLite3();
 private:
 	sqlite3 *m_db;

db.h

@@ -15,6 +15,7 @@ class DB {
 public:
 	virtual std::vector<int64_t> getBlockPos()=0;
 	virtual DBBlockList getBlocksOnZ(int zPos)=0;
+	virtual DBBlock getBlockOnPos(int64_t iPos)=0;
 };
 
 #endif // _DB_H

mapper.cpp

@@ -114,6 +114,7 @@ int main(int argc, char *argv[])
 				break;
 			case 'v':
 				generator.verboseCoordinates = true;
+				generator.verboseStatistics = true;
 				break;
 			case 'H':
 				generator.setShading(false);