From c692d3d6526c6838b878851623d646279a53dfe7 Mon Sep 17 00:00:00 2001 From: Rogier Date: Tue, 6 May 2014 16:20:22 +0200 Subject: [PATCH] Improvements to database code & some small optimisations --- TileGenerator.cpp | 36 ++---------------- TileGenerator.h | 3 -- db-leveldb.cpp | 24 +++++------- db-leveldb.h | 5 ++- db-sqlite3.cpp | 93 ++++++++++------------------------------------- db-sqlite3.h | 20 +++++----- db.h | 13 ++++--- 7 files changed, 53 insertions(+), 141 deletions(-) diff --git a/TileGenerator.cpp b/TileGenerator.cpp index cd4b9c1..f3466ac 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -436,12 +436,12 @@ void TileGenerator::loadBlocks() mapZMax = -INT_MIN/16+1; geomYMin = INT_MAX/16-1; geomYMax = -INT_MIN/16+1; - std::vector vec = m_db->getBlockPos(); + const DB::BlockPosList &blocks = m_db->getBlockPos(); world_blocks = 0; map_blocks = 0; - for(std::vector::iterator it = vec.begin(); it != vec.end(); ++it) { + for(DB::BlockPosList::const_iterator it = blocks.begin(); it != blocks.end(); ++it) { world_blocks ++; - BlockPos pos = decodeBlockPos(*it); + BlockPos pos = *it; if (pos.x < mapXMin) { mapXMin = pos.x; } @@ -807,34 +807,6 @@ void TileGenerator::createImage() } } -TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos) -{ - DBBlock in = m_db->getBlockOnPos(pos.x, pos.y, pos.z); - Block out(pos,(const unsigned char *)""); - - if (!in.second.empty()) { - out = Block(decodeBlockPos(in.first), in.second); - // Verify. Just to be sure... - if (pos.x != out.first.x || pos.y != out.first.y || pos.z != out.first.z) { - std::ostringstream oss; - oss << "Got unexpexted block: " - << out.first.x << "," << out.first.y << "," << out.first.z - << " from database. Requested: " - << pos.x << "," << pos.y << "," << pos.z; - throw std::runtime_error(oss.str()); - } - } - else { - // I can't imagine this to be possible... - std::ostringstream oss; - oss << "Failed to get block: " - << pos.x << "," << pos.y << "," << pos.z - << " from database."; - throw std::runtime_error(oss.str()); - } - return out; -} - void TileGenerator::renderMap() { int blocks_rendered = 0; @@ -863,7 +835,7 @@ void TileGenerator::renderMap() else if (allReaded) { continue; } - Block block = getBlockOnPos(pos); + DB::Block block = m_db->getBlockOnPos(pos); if (!block.second.empty()) { const unsigned char *data = block.second.c_str(); size_t length = block.second.length(); diff --git a/TileGenerator.h b/TileGenerator.h index fe6be08..46289b4 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -35,8 +35,6 @@ class TileGenerator { private: typedef std::map ColorMap; - typedef std::pair Block; - typedef std::list BlockList; public: TileGenerator(); @@ -91,7 +89,6 @@ private: bool ascending); void renderMap(); std::list getZValueList() const; - Block getBlockOnPos(BlockPos pos); void pushPixelRows(int zPosLimit); void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version); void renderScale(); diff --git a/db-leveldb.cpp b/db-leveldb.cpp index fb99e16..4324501 100644 --- a/db-leveldb.cpp +++ b/db-leveldb.cpp @@ -47,34 +47,30 @@ int DBLevelDB::getBlocksUnCachedCount(void) return m_blocksUnCachedCount; } -std::vector DBLevelDB::getBlockPos() { - std::vector vec; +const DB::BlockPosList &DBLevelDB::getBlockPos() { + m_blockPosList.clear(); leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { - vec.push_back(stoi64(it->key().ToString())); + m_blockPosList.push_back(stoi64(it->key().ToString())); } delete it; - return vec; + return m_blockPosList; } -DBBlock DBLevelDB::getBlockOnPos(int x, int y, int z) +DB::Block DBLevelDB::getBlockOnPos(const BlockPos &pos) { - int64_t iPos; - DBBlock block(0,(const unsigned char *)""); std::string datastr; leveldb::Status status; - iPos = static_cast(x); - iPos += static_cast(y) << 12; - iPos += static_cast(z) << 24; - - status = m_db->Get(leveldb::ReadOptions(), i64tos(iPos), &datastr); + status = m_db->Get(leveldb::ReadOptions(), i64tos(pos.databasePos()), &datastr); if(status.ok()) { - block = DBBlock( iPos, ustring( (const unsigned char*) datastr.c_str(), datastr.size() ) ); m_blocksReadCount++; m_blocksUnCachedCount++; + return Block(pos, ustring(reinterpret_cast(datastr.c_str()), datastr.size())); + } + else { + return Block(pos, ustring(reinterpret_cast(""))); } - return block; } diff --git a/db-leveldb.h b/db-leveldb.h index 052033e..6335eb8 100644 --- a/db-leveldb.h +++ b/db-leveldb.h @@ -11,14 +11,15 @@ public: virtual int getBlocksUnCachedCount(void); virtual int getBlocksCachedCount(void); virtual int getBlocksReadCount(void); - virtual std::vector getBlockPos(); - virtual DBBlock getBlockOnPos(int x, int y, int z); + virtual const BlockPosList &getBlockPos(); + virtual Block getBlockOnPos(const BlockPos &pos); ~DBLevelDB(); private: int m_blocksReadCount; int m_blocksCachedCount; int m_blocksUnCachedCount; leveldb::DB *m_db; + BlockPosList m_blockPosList; }; #endif // _DB_LEVELDB_H diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index e00de74..e849ac4 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -42,8 +42,8 @@ int DBSQLite3::getBlocksUnCachedCount(void) return m_blocksUnCachedCount; } -std::vector DBSQLite3::getBlockPos() { - std::vector vec; +const DB::BlockPosList &DBSQLite3::getBlockPos() { + m_BlockPosList.clear(); std::string sql = "SELECT pos FROM blocks"; if (m_blockPosListStatement || sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blockPosListStatement, 0) == SQLITE_OK) { int result = 0; @@ -51,7 +51,7 @@ std::vector DBSQLite3::getBlockPos() { result = sqlite3_step(m_blockPosListStatement); if(result == SQLITE_ROW) { sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0); - vec.push_back(blocknum); + m_BlockPosList.push_back(blocknum); } else if (result == SQLITE_BUSY) // Wait some time and try again usleep(10000); else @@ -62,7 +62,7 @@ std::vector DBSQLite3::getBlockPos() { throw std::runtime_error("Failed to get list of MapBlocks"); } sqlite3_reset(m_blockPosListStatement); - return vec; + return m_BlockPosList; } void DBSQLite3::prepareBlocksOnZStatement(void) @@ -82,41 +82,15 @@ void DBSQLite3::prepareBlockOnPosStatement(void) } } -// Apparently, this attempt at being smart, is actually quite inefficient for sqlite. -// -// For larger subsections of the map, it performs much worse than caching an entire -// world row (i.e. z coordinate). In cases where it *is* more efficient, no caching is -// much more efficient still. -// -// It seems that any computation on pos severely affects the performance (?)... -// -// For the moment, this function is not used. -void DBSQLite3::prepareBlocksYRangeStatement(void) -{ - // This one seems to perform best: - std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3)&4095 == 0 AND (pos-?3 BETWEEN ?4 AND ?5)"; - // These perform worse: - //std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0"; - //std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5)"; - //std::string sql = "SELECT pos, data FROM (select pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0"; - //std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5) AND (pos3)&4095 == 0"; - //std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5)"; - if (!m_blocksYRangeStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blocksYRangeStatement, 0) != SQLITE_OK) { - throw std::runtime_error("Failed to prepare SQL statement (blocksYRangeStatement)"); - } -} - void DBSQLite3::cacheBlocksOnZRaw(int zPos) { prepareBlocksOnZStatement(); - DBBlockList blocks; - sqlite3_int64 psMin; sqlite3_int64 psMax; - psMin = (static_cast(zPos) * 16777216l) - 0x800000; - psMax = (static_cast(zPos) * 16777216l) + 0x7fffff; + psMin = (static_cast(zPos) * 0x1000000L) - 0x800000; + psMax = (static_cast(zPos) * 0x1000000L) + 0x7fffff; sqlite3_bind_int64(m_blocksOnZStatement, 1, psMin); sqlite3_bind_int64(m_blocksOnZStatement, 2, psMax); @@ -124,24 +98,22 @@ void DBSQLite3::cacheBlocksOnZRaw(int zPos) cacheBlocks(m_blocksOnZStatement); } -DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos) +DB::Block DBSQLite3::getBlockOnPosRaw(const BlockPos &pos) { prepareBlockOnPosStatement(); - DBBlock block(0,(const unsigned char *)""); + Block block(pos,reinterpret_cast("")); int result = 0; - sqlite3_bind_int64(m_blockOnPosStatement, 1, psPos); + sqlite3_bind_int64(m_blockOnPosStatement, 1, pos.databasePos()); while (true) { result = sqlite3_step(m_blockOnPosStatement); if(result == SQLITE_ROW) { - sqlite3_int64 blocknum = sqlite3_column_int64(m_blockOnPosStatement, 0); const unsigned char *data = reinterpret_cast(sqlite3_column_blob(m_blockOnPosStatement, 1)); int size = sqlite3_column_bytes(m_blockOnPosStatement, 1); - block = DBBlock(blocknum, ustring(data, size)); + block = Block(pos, ustring(data, size)); m_blocksUnCachedCount++; - //std::cerr << "Read block " << blocknum << " from database" << std::endl; break; } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); @@ -154,26 +126,6 @@ DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos) return block; } -void DBSQLite3::cacheBlocksYRangeRaw(int x, int y, int z) -{ - prepareBlocksYRangeStatement(); - - sqlite3_int64 psZPosFrom = (static_cast(z) << 24) - 0x800000; - sqlite3_int64 psZPosTo = (static_cast(z) << 24) + 0x7fffff; - sqlite3_int64 psPosZero = static_cast(x); - psPosZero += static_cast(z) << 24; - sqlite3_int64 psYPosFrom = 0; - sqlite3_int64 psYPosTo = static_cast(y) << 12; - - sqlite3_bind_int64(m_blocksYRangeStatement, 1, psZPosFrom); - sqlite3_bind_int64(m_blocksYRangeStatement, 2, psZPosTo); - sqlite3_bind_int64(m_blocksYRangeStatement, 3, psPosZero); - sqlite3_bind_int64(m_blocksYRangeStatement, 4, psYPosFrom); - sqlite3_bind_int64(m_blocksYRangeStatement, 5, psYPosTo); - - cacheBlocks(m_blocksYRangeStatement); -} - void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement) { int result = 0; @@ -183,9 +135,8 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement) sqlite3_int64 blocknum = sqlite3_column_int64(SQLstatement, 0); const unsigned char *data = reinterpret_cast(sqlite3_column_blob(SQLstatement, 1)); int size = sqlite3_column_bytes(SQLstatement, 1); - m_blockCache[blocknum] = DBBlock(blocknum, ustring(data, size)); + m_blockCache[blocknum] = ustring(data, size); m_blocksCachedCount++; - //std::cerr << "Cache block " << blocknum << " from database" << std::endl; } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); } else { @@ -195,36 +146,30 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement) sqlite3_reset(SQLstatement); } -DBBlock DBSQLite3::getBlockOnPos(int x, int y, int z) +DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos) { - sqlite3_int64 psPos; - psPos = static_cast(x); - psPos += static_cast(y) << 12; - psPos += static_cast(z) << 24; - //std::cerr << "Block " << x << "," << y << "," << z << " -> " << psPos << std::endl; - m_blocksReadCount++; BlockCache::const_iterator DBBlockSearch; - DBBlockSearch = m_blockCache.find(psPos); + DBBlockSearch = m_blockCache.find(pos.databasePos()); if (DBBlockSearch == m_blockCache.end()) { if (cacheWorldRow) { m_blockCache.clear(); - cacheBlocksOnZRaw(z); - DBBlockSearch = m_blockCache.find(psPos); + cacheBlocksOnZRaw(pos.z); + DBBlockSearch = m_blockCache.find(pos.databasePos()); if (DBBlockSearch != m_blockCache.end()) { - return DBBlockSearch->second; + return Block(pos, DBBlockSearch->second); } else { - return DBBlock(0, (const unsigned char *)""); + return Block(pos,reinterpret_cast("")); } } else { - return getBlockOnPosRaw(psPos); + return getBlockOnPosRaw(pos); } } else { - return DBBlockSearch->second; + return Block(pos, DBBlockSearch->second); } } diff --git a/db-sqlite3.h b/db-sqlite3.h index a467aef..4495051 100644 --- a/db-sqlite3.h +++ b/db-sqlite3.h @@ -3,7 +3,7 @@ #include "db.h" #include -#if _cplusplus == 201103L +#if _cplusplus >= 201103L #include #else #include @@ -11,11 +11,13 @@ #include #include +#include "types.h" + class DBSQLite3 : public DB { -#if _cplusplus == 201103L - typedef std::unordered_map BlockCache; +#if _cplusplus >= 201103L + typedef std::unordered_map BlockCache; #else - typedef std::map BlockCache; + typedef std::map BlockCache; #endif public: bool cacheWorldRow; @@ -23,8 +25,8 @@ public: virtual int getBlocksUnCachedCount(void); virtual int getBlocksCachedCount(void); virtual int getBlocksReadCount(void); - virtual std::vector getBlockPos(); - virtual DBBlock getBlockOnPos(int x, int y, int z); + virtual const BlockPosList &getBlockPos(); + virtual Block getBlockOnPos(const BlockPos &pos); ~DBSQLite3(); private: int m_blocksReadCount; @@ -34,16 +36,14 @@ private: sqlite3_stmt *m_blockPosListStatement; sqlite3_stmt *m_blocksOnZStatement; sqlite3_stmt *m_blockOnPosStatement; - sqlite3_stmt *m_blocksYRangeStatement; std::ostringstream m_getBlockSetStatementBlocks; BlockCache m_blockCache; + BlockPosList m_BlockPosList; void prepareBlocksOnZStatement(void); void prepareBlockOnPosStatement(void); - void prepareBlocksYRangeStatement(void); - void cacheBlocksYRangeRaw(int x, int y, int z); void cacheBlocksOnZRaw(int zPos); - DBBlock getBlockOnPosRaw(sqlite3_int64 psPos); + Block getBlockOnPosRaw(const BlockPos &pos); void cacheBlocks(sqlite3_stmt *SQLstatement); }; diff --git a/db.h b/db.h index 296a53b..f3982e8 100644 --- a/db.h +++ b/db.h @@ -3,21 +3,22 @@ #include #include -#include #include #include -// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit) -typedef std::pair > DBBlock; -typedef std::list DBBlockList; +#include "types.h" +#include "BlockPos.h" + class DB { public: - virtual std::vector getBlockPos()=0; + typedef std::pair Block; + typedef std::vector BlockPosList; + virtual const BlockPosList &getBlockPos()=0; virtual int getBlocksUnCachedCount(void)=0; virtual int getBlocksCachedCount(void)=0; virtual int getBlocksReadCount(void)=0; - virtual DBBlock getBlockOnPos(int x, int y, int z)=0; + virtual Block getBlockOnPos(const BlockPos &pos)=0; }; #endif // _DB_H