Improvements to database code & some small optimisations

TileGenerator.cpp

@@ -436,12 +436,12 @@ void TileGenerator::loadBlocks()
 	mapZMax = -INT_MIN/16+1;
 	geomYMin = INT_MAX/16-1;
 	geomYMax = -INT_MIN/16+1;
-	std::vector<int64_t> vec = m_db->getBlockPos();
+	const DB::BlockPosList &blocks = m_db->getBlockPos();
 	world_blocks = 0;
 	map_blocks = 0;
-	for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
+	for(DB::BlockPosList::const_iterator it = blocks.begin(); it != blocks.end(); ++it) {
 		world_blocks ++;
-		BlockPos pos = decodeBlockPos(*it);
+		BlockPos pos = *it;
 		if (pos.x < mapXMin) {
 			mapXMin = pos.x;
 		}
@@ -807,34 +807,6 @@ void TileGenerator::createImage()
 	}
 }
 
-TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
-{
-	DBBlock in = m_db->getBlockOnPos(pos.x, pos.y, pos.z);
-	Block out(pos,(const unsigned char *)"");
-
-	if (!in.second.empty()) {
-		out = Block(decodeBlockPos(in.first), in.second);
-		// Verify. Just to be sure...
-		if (pos.x != out.first.x || pos.y != out.first.y || pos.z != out.first.z) {
-			std::ostringstream oss;
-			oss << "Got unexpexted block: "
-			    << out.first.x << "," << out.first.y << "," << out.first.z
-			    << " from database. Requested: "
-			    << pos.x << "," << pos.y << "," << pos.z;
-			throw std::runtime_error(oss.str());
-		}
-	}
-	else {
-		// I can't imagine this to be possible...
-		std::ostringstream oss;
-		oss << "Failed to get block: "
-		    << pos.x << "," << pos.y << "," << pos.z
-		    << " from database.";
-		throw std::runtime_error(oss.str());
-	}
-	return out;
-}
-
 void TileGenerator::renderMap()
 {
 	int blocks_rendered = 0;
@@ -863,7 +835,7 @@ void TileGenerator::renderMap()
 		else if (allReaded) {
 			continue;
 		}
-		Block block = getBlockOnPos(pos);
+		DB::Block block = m_db->getBlockOnPos(pos);
 		if (!block.second.empty()) {
 			const unsigned char *data = block.second.c_str();
 			size_t length = block.second.length();

TileGenerator.h

@@ -35,8 +35,6 @@ class TileGenerator
 {
 private:
 	typedef std::map<std::string, ColorEntry> ColorMap;
-	typedef std::pair<BlockPos, ustring> Block;
-	typedef std::list<Block> BlockList;
 
 public:
 	TileGenerator();
@@ -91,7 +89,6 @@ private:
                 bool ascending);
 	void renderMap();
 	std::list<int> getZValueList() const;
-	Block getBlockOnPos(BlockPos pos);
 	void pushPixelRows(int zPosLimit);
 	void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
 	void renderScale();

db-leveldb.cpp

@@ -47,34 +47,30 @@ int DBLevelDB::getBlocksUnCachedCount(void)
 	return m_blocksUnCachedCount;
 }
 
-std::vector<int64_t> DBLevelDB::getBlockPos() {
+const DB::BlockPosList &DBLevelDB::getBlockPos() {
-	std::vector<int64_t> vec;
+	m_blockPosList.clear();
 	leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
-		vec.push_back(stoi64(it->key().ToString()));
+		m_blockPosList.push_back(stoi64(it->key().ToString()));
 	}
 	delete it;
-	return vec;
+	return m_blockPosList;
 }
 
-DBBlock DBLevelDB::getBlockOnPos(int x, int y, int z)
+DB::Block DBLevelDB::getBlockOnPos(const BlockPos &pos)
 {
-	int64_t iPos;
-	DBBlock block(0,(const unsigned char *)"");
 	std::string datastr;
 	leveldb::Status status;
 
-	iPos =  static_cast<int64_t>(x);
+	status = m_db->Get(leveldb::ReadOptions(), i64tos(pos.databasePos()), &datastr);
-	iPos += static_cast<int64_t>(y) << 12;
-	iPos += static_cast<int64_t>(z) << 24;
-
-	status = m_db->Get(leveldb::ReadOptions(), i64tos(iPos), &datastr);
 	if(status.ok()) {
-		block = DBBlock( iPos, ustring( (const unsigned char*) datastr.c_str(), datastr.size() ) );
 		m_blocksReadCount++;
 		m_blocksUnCachedCount++;
+		return Block(pos, ustring(reinterpret_cast<const unsigned char *>(datastr.c_str()), datastr.size()));
+	}
+	else {
+		return Block(pos, ustring(reinterpret_cast<const unsigned char *>("")));
 	}
 
-	return block;
 }
 

db-leveldb.h

@@ -11,14 +11,15 @@ public:
 	virtual int getBlocksUnCachedCount(void);
 	virtual int getBlocksCachedCount(void);
 	virtual int getBlocksReadCount(void);
-	virtual std::vector<int64_t> getBlockPos();
+	virtual const BlockPosList &getBlockPos();
-	virtual DBBlock getBlockOnPos(int x, int y, int z);
+	virtual Block getBlockOnPos(const BlockPos &pos);
 	~DBLevelDB();
 private:
 	int m_blocksReadCount;
 	int m_blocksCachedCount;
 	int m_blocksUnCachedCount;
 	leveldb::DB *m_db;
+	BlockPosList m_blockPosList;
 };
 
 #endif // _DB_LEVELDB_H

db-sqlite3.cpp

@@ -42,8 +42,8 @@ int DBSQLite3::getBlocksUnCachedCount(void)
 	return m_blocksUnCachedCount;
 }
 
-std::vector<int64_t> DBSQLite3::getBlockPos() {
+const DB::BlockPosList &DBSQLite3::getBlockPos() {
-	std::vector<int64_t> vec;
+	m_BlockPosList.clear();
 	std::string sql = "SELECT pos FROM blocks";
 	if (m_blockPosListStatement || sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blockPosListStatement, 0) == SQLITE_OK) {
 		int result = 0;
@@ -51,7 +51,7 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
 			result = sqlite3_step(m_blockPosListStatement);
 			if(result == SQLITE_ROW) {
 				sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
-				vec.push_back(blocknum);
+				m_BlockPosList.push_back(blocknum);
 			} else if (result == SQLITE_BUSY) // Wait some time and try again
 				usleep(10000);
 			else
@@ -62,7 +62,7 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
 		throw std::runtime_error("Failed to get list of MapBlocks");
 	}
 	sqlite3_reset(m_blockPosListStatement);
-	return vec;
+	return m_BlockPosList;
 }
 
 void DBSQLite3::prepareBlocksOnZStatement(void)
@@ -82,41 +82,15 @@ void DBSQLite3::prepareBlockOnPosStatement(void)
 	}
 }
 
-// Apparently, this attempt at being smart, is actually quite inefficient for sqlite.
-//
-// For larger subsections of the map, it performs much worse than caching an entire
-// world row (i.e. z coordinate).  In cases where it *is* more efficient, no caching is
-// much more efficient still.
-//
-// It seems that any computation on pos severely affects the performance (?)...
-//
-// For the moment, this function is not used.
-void DBSQLite3::prepareBlocksYRangeStatement(void)
-{
-	// This one seems to perform best:
-	std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3)&4095 == 0 AND (pos-?3 BETWEEN ?4 AND ?5)";
-	// These perform worse:
-	//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
-	//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5)";
-	//std::string sql = "SELECT pos, data FROM (select pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
-	//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5) AND (pos3)&4095 == 0";
-	//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5)";
-	if (!m_blocksYRangeStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blocksYRangeStatement, 0) != SQLITE_OK) {
-		throw std::runtime_error("Failed to prepare SQL statement (blocksYRangeStatement)");
-	}
-}
-
 void DBSQLite3::cacheBlocksOnZRaw(int zPos)
 {
 	prepareBlocksOnZStatement();
 
-	DBBlockList blocks;
-
 	sqlite3_int64 psMin;
 	sqlite3_int64 psMax;
 
-	psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
+	psMin = (static_cast<sqlite3_int64>(zPos) * 0x1000000L) - 0x800000;
-	psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
+	psMax = (static_cast<sqlite3_int64>(zPos) * 0x1000000L) + 0x7fffff;
 
 	sqlite3_bind_int64(m_blocksOnZStatement, 1, psMin);
 	sqlite3_bind_int64(m_blocksOnZStatement, 2, psMax);
@@ -124,24 +98,22 @@ void DBSQLite3::cacheBlocksOnZRaw(int zPos)
 	cacheBlocks(m_blocksOnZStatement);
 }
 
-DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos)
+DB::Block DBSQLite3::getBlockOnPosRaw(const BlockPos &pos)
 {
 	prepareBlockOnPosStatement();
 
-	DBBlock block(0,(const unsigned char *)"");
+	Block block(pos,reinterpret_cast<const unsigned char *>(""));
 	int result = 0;
 
-	sqlite3_bind_int64(m_blockOnPosStatement, 1, psPos);
+	sqlite3_bind_int64(m_blockOnPosStatement, 1, pos.databasePos());
 
 	while (true) {
 		result = sqlite3_step(m_blockOnPosStatement);
 		if(result == SQLITE_ROW) {
-			sqlite3_int64 blocknum = sqlite3_column_int64(m_blockOnPosStatement, 0);
 			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_blockOnPosStatement, 1));
 			int size = sqlite3_column_bytes(m_blockOnPosStatement, 1);
-			block = DBBlock(blocknum, ustring(data, size));
+			block = Block(pos, ustring(data, size));
 			m_blocksUnCachedCount++;
-			//std::cerr << "Read block " << blocknum << " from database" << std::endl;
 			break;
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
@@ -154,26 +126,6 @@ DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos)
 	return block;
 }
 
-void DBSQLite3::cacheBlocksYRangeRaw(int x, int y, int z)
-{
-	prepareBlocksYRangeStatement();
-
-	sqlite3_int64 psZPosFrom = (static_cast<sqlite3_int64>(z) << 24) - 0x800000;
-	sqlite3_int64 psZPosTo   = (static_cast<sqlite3_int64>(z) << 24) + 0x7fffff;
-	sqlite3_int64 psPosZero  =  static_cast<sqlite3_int64>(x);
-	              psPosZero  += static_cast<sqlite3_int64>(z) << 24;
-	sqlite3_int64 psYPosFrom =  0;
-	sqlite3_int64 psYPosTo   =  static_cast<sqlite3_int64>(y) << 12;
-
-	sqlite3_bind_int64(m_blocksYRangeStatement, 1, psZPosFrom);
-	sqlite3_bind_int64(m_blocksYRangeStatement, 2, psZPosTo);
-	sqlite3_bind_int64(m_blocksYRangeStatement, 3, psPosZero);
-	sqlite3_bind_int64(m_blocksYRangeStatement, 4, psYPosFrom);
-	sqlite3_bind_int64(m_blocksYRangeStatement, 5, psYPosTo);
-
-	cacheBlocks(m_blocksYRangeStatement);
-}
-
 void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
 {
 	int result = 0;
@@ -183,9 +135,8 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
 			sqlite3_int64 blocknum = sqlite3_column_int64(SQLstatement, 0);
 			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(SQLstatement, 1));
 			int size = sqlite3_column_bytes(SQLstatement, 1);
-			m_blockCache[blocknum] = DBBlock(blocknum, ustring(data, size));
+			m_blockCache[blocknum] = ustring(data, size);
 			m_blocksCachedCount++;
-			//std::cerr << "Cache block " << blocknum << " from database" << std::endl;
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
 		} else {
@@ -195,36 +146,30 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
 	sqlite3_reset(SQLstatement);
 }
 
-DBBlock DBSQLite3::getBlockOnPos(int x, int y, int z)
+DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
 {
-	sqlite3_int64 psPos;
-	psPos =  static_cast<sqlite3_int64>(x);
-	psPos += static_cast<sqlite3_int64>(y) << 12;
-	psPos += static_cast<sqlite3_int64>(z) << 24;
-	//std::cerr << "Block " << x << "," << y << "," << z << " -> " << psPos << std::endl;
-
 	m_blocksReadCount++;
 
 	BlockCache::const_iterator DBBlockSearch;
-	DBBlockSearch = m_blockCache.find(psPos);
+	DBBlockSearch = m_blockCache.find(pos.databasePos());
 	if (DBBlockSearch == m_blockCache.end()) {
 		if (cacheWorldRow) {
 			m_blockCache.clear();
-			cacheBlocksOnZRaw(z);
+			cacheBlocksOnZRaw(pos.z);
-			DBBlockSearch = m_blockCache.find(psPos);
+			DBBlockSearch = m_blockCache.find(pos.databasePos());
 			if (DBBlockSearch != m_blockCache.end()) {
-				return DBBlockSearch->second;
+				return Block(pos, DBBlockSearch->second);
 			}
 			else {
-				return DBBlock(0, (const unsigned char *)"");
+				return Block(pos,reinterpret_cast<const unsigned char *>(""));
 			}
 		}
 		else {
-			return getBlockOnPosRaw(psPos);
+			return getBlockOnPosRaw(pos);
 		}
 	}
 	else {
-		return DBBlockSearch->second;
+		return Block(pos, DBBlockSearch->second);
 	}
 }
 

db-sqlite3.h

@@ -3,7 +3,7 @@
 
 #include "db.h"
 #include <sqlite3.h>
-#if _cplusplus == 201103L
+#if _cplusplus >= 201103L
 #include <unordered_map>
 #else
 #include <map>
@@ -11,11 +11,13 @@
 #include <string>
 #include <sstream>
 
+#include "types.h"
+
 class DBSQLite3 : public DB {
-#if _cplusplus == 201103L
+#if _cplusplus >= 201103L
-	typedef std::unordered_map<uint64_t, DBBlock>  BlockCache;
+	typedef std::unordered_map<int64_t, ustring>  BlockCache;
 #else
-	typedef std::map<uint64_t, DBBlock>  BlockCache;
+	typedef std::map<int64_t, ustring>  BlockCache;
 #endif
 public:
 	bool cacheWorldRow;
@@ -23,8 +25,8 @@ public:
 	virtual int getBlocksUnCachedCount(void);
 	virtual int getBlocksCachedCount(void);
 	virtual int getBlocksReadCount(void);
-	virtual std::vector<int64_t> getBlockPos();
+	virtual const BlockPosList &getBlockPos();
-	virtual DBBlock getBlockOnPos(int x, int y, int z);
+	virtual Block getBlockOnPos(const BlockPos &pos);
 	~DBSQLite3();
 private:
 	int m_blocksReadCount;
@@ -34,16 +36,14 @@ private:
 	sqlite3_stmt *m_blockPosListStatement;
 	sqlite3_stmt *m_blocksOnZStatement;
 	sqlite3_stmt *m_blockOnPosStatement;
-	sqlite3_stmt *m_blocksYRangeStatement;
 	std::ostringstream  m_getBlockSetStatementBlocks;
 	BlockCache  m_blockCache;
+	BlockPosList m_BlockPosList;
 
 	void prepareBlocksOnZStatement(void);
 	void prepareBlockOnPosStatement(void);
-	void prepareBlocksYRangeStatement(void);
-	void cacheBlocksYRangeRaw(int x, int y, int z);
 	void cacheBlocksOnZRaw(int zPos);
-	DBBlock getBlockOnPosRaw(sqlite3_int64 psPos);
+	Block getBlockOnPosRaw(const BlockPos &pos);
 	void cacheBlocks(sqlite3_stmt *SQLstatement);
 };
 

db.h

@@ -3,21 +3,22 @@
 
 #include <stdint.h>
 #include <vector>
-#include <list>
 #include <string>
 #include <utility>
 
-// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit)
+#include "types.h"
-typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
+#include "BlockPos.h"
-typedef std::list<DBBlock> DBBlockList;
+
 
 class DB {
 public:
-	virtual std::vector<int64_t> getBlockPos()=0;
+	typedef std::pair<BlockPos, ustring> Block;
+	typedef std::vector<int64_t>  BlockPosList;
+	virtual const BlockPosList &getBlockPos()=0;
 	virtual int getBlocksUnCachedCount(void)=0;
 	virtual int getBlocksCachedCount(void)=0;
 	virtual int getBlocksReadCount(void)=0;
-	virtual DBBlock getBlockOnPos(int x, int y, int z)=0;
+	virtual Block getBlockOnPos(const BlockPos &pos)=0;
 };
 
 #endif // _DB_H