Improvements to database code & some small optimisations
This commit is contained in:
Rogier
parent
bbf92d2ffa
commit
c692d3d652
@ -436,12 +436,12 @@ void TileGenerator::loadBlocks()
|
||||
mapZMax = -INT_MIN/16+1;
|
||||
geomYMin = INT_MAX/16-1;
|
||||
geomYMax = -INT_MIN/16+1;
|
||||
std::vector<int64_t> vec = m_db->getBlockPos();
|
||||
const DB::BlockPosList &blocks = m_db->getBlockPos();
|
||||
world_blocks = 0;
|
||||
map_blocks = 0;
|
||||
for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
|
||||
for(DB::BlockPosList::const_iterator it = blocks.begin(); it != blocks.end(); ++it) {
|
||||
world_blocks ++;
|
||||
BlockPos pos = decodeBlockPos(*it);
|
||||
BlockPos pos = *it;
|
||||
if (pos.x < mapXMin) {
|
||||
mapXMin = pos.x;
|
||||
}
|
||||
@ -807,34 +807,6 @@ void TileGenerator::createImage()
|
||||
}
|
||||
}
|
||||
|
||||
TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
|
||||
{
|
||||
DBBlock in = m_db->getBlockOnPos(pos.x, pos.y, pos.z);
|
||||
Block out(pos,(const unsigned char *)"");
|
||||
|
||||
if (!in.second.empty()) {
|
||||
out = Block(decodeBlockPos(in.first), in.second);
|
||||
// Verify. Just to be sure...
|
||||
if (pos.x != out.first.x || pos.y != out.first.y || pos.z != out.first.z) {
|
||||
std::ostringstream oss;
|
||||
oss << "Got unexpexted block: "
|
||||
<< out.first.x << "," << out.first.y << "," << out.first.z
|
||||
<< " from database. Requested: "
|
||||
<< pos.x << "," << pos.y << "," << pos.z;
|
||||
throw std::runtime_error(oss.str());
|
||||
}
|
||||
}
|
||||
else {
|
||||
// I can't imagine this to be possible...
|
||||
std::ostringstream oss;
|
||||
oss << "Failed to get block: "
|
||||
<< pos.x << "," << pos.y << "," << pos.z
|
||||
<< " from database.";
|
||||
throw std::runtime_error(oss.str());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void TileGenerator::renderMap()
|
||||
{
|
||||
int blocks_rendered = 0;
|
||||
@ -863,7 +835,7 @@ void TileGenerator::renderMap()
|
||||
else if (allReaded) {
|
||||
continue;
|
||||
}
|
||||
Block block = getBlockOnPos(pos);
|
||||
DB::Block block = m_db->getBlockOnPos(pos);
|
||||
if (!block.second.empty()) {
|
||||
const unsigned char *data = block.second.c_str();
|
||||
size_t length = block.second.length();
|
||||
|
||||
@ -35,8 +35,6 @@ class TileGenerator
|
||||
{
|
||||
private:
|
||||
typedef std::map<std::string, ColorEntry> ColorMap;
|
||||
typedef std::pair<BlockPos, ustring> Block;
|
||||
typedef std::list<Block> BlockList;
|
||||
|
||||
public:
|
||||
TileGenerator();
|
||||
@ -91,7 +89,6 @@ private:
|
||||
bool ascending);
|
||||
void renderMap();
|
||||
std::list<int> getZValueList() const;
|
||||
Block getBlockOnPos(BlockPos pos);
|
||||
void pushPixelRows(int zPosLimit);
|
||||
void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
|
||||
void renderScale();
|
||||
|
||||
@ -47,34 +47,30 @@ int DBLevelDB::getBlocksUnCachedCount(void)
|
||||
return m_blocksUnCachedCount;
|
||||
}
|
||||
|
||||
std::vector<int64_t> DBLevelDB::getBlockPos() {
|
||||
std::vector<int64_t> vec;
|
||||
const DB::BlockPosList &DBLevelDB::getBlockPos() {
|
||||
m_blockPosList.clear();
|
||||
leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
vec.push_back(stoi64(it->key().ToString()));
|
||||
m_blockPosList.push_back(stoi64(it->key().ToString()));
|
||||
}
|
||||
delete it;
|
||||
return vec;
|
||||
return m_blockPosList;
|
||||
}
|
||||
|
||||
DBBlock DBLevelDB::getBlockOnPos(int x, int y, int z)
|
||||
DB::Block DBLevelDB::getBlockOnPos(const BlockPos &pos)
|
||||
{
|
||||
int64_t iPos;
|
||||
DBBlock block(0,(const unsigned char *)"");
|
||||
std::string datastr;
|
||||
leveldb::Status status;
|
||||
|
||||
iPos = static_cast<int64_t>(x);
|
||||
iPos += static_cast<int64_t>(y) << 12;
|
||||
iPos += static_cast<int64_t>(z) << 24;
|
||||
|
||||
status = m_db->Get(leveldb::ReadOptions(), i64tos(iPos), &datastr);
|
||||
status = m_db->Get(leveldb::ReadOptions(), i64tos(pos.databasePos()), &datastr);
|
||||
if(status.ok()) {
|
||||
block = DBBlock( iPos, ustring( (const unsigned char*) datastr.c_str(), datastr.size() ) );
|
||||
m_blocksReadCount++;
|
||||
m_blocksUnCachedCount++;
|
||||
return Block(pos, ustring(reinterpret_cast<const unsigned char *>(datastr.c_str()), datastr.size()));
|
||||
}
|
||||
else {
|
||||
return Block(pos, ustring(reinterpret_cast<const unsigned char *>("")));
|
||||
}
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
|
||||
@ -11,14 +11,15 @@ public:
|
||||
virtual int getBlocksUnCachedCount(void);
|
||||
virtual int getBlocksCachedCount(void);
|
||||
virtual int getBlocksReadCount(void);
|
||||
virtual std::vector<int64_t> getBlockPos();
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z);
|
||||
virtual const BlockPosList &getBlockPos();
|
||||
virtual Block getBlockOnPos(const BlockPos &pos);
|
||||
~DBLevelDB();
|
||||
private:
|
||||
int m_blocksReadCount;
|
||||
int m_blocksCachedCount;
|
||||
int m_blocksUnCachedCount;
|
||||
leveldb::DB *m_db;
|
||||
BlockPosList m_blockPosList;
|
||||
};
|
||||
|
||||
#endif // _DB_LEVELDB_H
|
||||
|
||||
@ -42,8 +42,8 @@ int DBSQLite3::getBlocksUnCachedCount(void)
|
||||
return m_blocksUnCachedCount;
|
||||
}
|
||||
|
||||
std::vector<int64_t> DBSQLite3::getBlockPos() {
|
||||
std::vector<int64_t> vec;
|
||||
const DB::BlockPosList &DBSQLite3::getBlockPos() {
|
||||
m_BlockPosList.clear();
|
||||
std::string sql = "SELECT pos FROM blocks";
|
||||
if (m_blockPosListStatement || sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blockPosListStatement, 0) == SQLITE_OK) {
|
||||
int result = 0;
|
||||
@ -51,7 +51,7 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
|
||||
result = sqlite3_step(m_blockPosListStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
|
||||
vec.push_back(blocknum);
|
||||
m_BlockPosList.push_back(blocknum);
|
||||
} else if (result == SQLITE_BUSY) // Wait some time and try again
|
||||
usleep(10000);
|
||||
else
|
||||
@ -62,7 +62,7 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
|
||||
throw std::runtime_error("Failed to get list of MapBlocks");
|
||||
}
|
||||
sqlite3_reset(m_blockPosListStatement);
|
||||
return vec;
|
||||
return m_BlockPosList;
|
||||
}
|
||||
|
||||
void DBSQLite3::prepareBlocksOnZStatement(void)
|
||||
@ -82,41 +82,15 @@ void DBSQLite3::prepareBlockOnPosStatement(void)
|
||||
}
|
||||
}
|
||||
|
||||
// Apparently, this attempt at being smart, is actually quite inefficient for sqlite.
|
||||
//
|
||||
// For larger subsections of the map, it performs much worse than caching an entire
|
||||
// world row (i.e. z coordinate). In cases where it *is* more efficient, no caching is
|
||||
// much more efficient still.
|
||||
//
|
||||
// It seems that any computation on pos severely affects the performance (?)...
|
||||
//
|
||||
// For the moment, this function is not used.
|
||||
void DBSQLite3::prepareBlocksYRangeStatement(void)
|
||||
{
|
||||
// This one seems to perform best:
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3)&4095 == 0 AND (pos-?3 BETWEEN ?4 AND ?5)";
|
||||
// These perform worse:
|
||||
//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5)";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5) AND (pos3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5)";
|
||||
if (!m_blocksYRangeStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blocksYRangeStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blocksYRangeStatement)");
|
||||
}
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocksOnZRaw(int zPos)
|
||||
{
|
||||
prepareBlocksOnZStatement();
|
||||
|
||||
DBBlockList blocks;
|
||||
|
||||
sqlite3_int64 psMin;
|
||||
sqlite3_int64 psMax;
|
||||
|
||||
psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
|
||||
psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
|
||||
psMin = (static_cast<sqlite3_int64>(zPos) * 0x1000000L) - 0x800000;
|
||||
psMax = (static_cast<sqlite3_int64>(zPos) * 0x1000000L) + 0x7fffff;
|
||||
|
||||
sqlite3_bind_int64(m_blocksOnZStatement, 1, psMin);
|
||||
sqlite3_bind_int64(m_blocksOnZStatement, 2, psMax);
|
||||
@ -124,24 +98,22 @@ void DBSQLite3::cacheBlocksOnZRaw(int zPos)
|
||||
cacheBlocks(m_blocksOnZStatement);
|
||||
}
|
||||
|
||||
DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos)
|
||||
DB::Block DBSQLite3::getBlockOnPosRaw(const BlockPos &pos)
|
||||
{
|
||||
prepareBlockOnPosStatement();
|
||||
|
||||
DBBlock block(0,(const unsigned char *)"");
|
||||
Block block(pos,reinterpret_cast<const unsigned char *>(""));
|
||||
int result = 0;
|
||||
|
||||
sqlite3_bind_int64(m_blockOnPosStatement, 1, psPos);
|
||||
sqlite3_bind_int64(m_blockOnPosStatement, 1, pos.databasePos());
|
||||
|
||||
while (true) {
|
||||
result = sqlite3_step(m_blockOnPosStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockOnPosStatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_blockOnPosStatement, 1));
|
||||
int size = sqlite3_column_bytes(m_blockOnPosStatement, 1);
|
||||
block = DBBlock(blocknum, ustring(data, size));
|
||||
block = Block(pos, ustring(data, size));
|
||||
m_blocksUnCachedCount++;
|
||||
//std::cerr << "Read block " << blocknum << " from database" << std::endl;
|
||||
break;
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
@ -154,26 +126,6 @@ DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos)
|
||||
return block;
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocksYRangeRaw(int x, int y, int z)
|
||||
{
|
||||
prepareBlocksYRangeStatement();
|
||||
|
||||
sqlite3_int64 psZPosFrom = (static_cast<sqlite3_int64>(z) << 24) - 0x800000;
|
||||
sqlite3_int64 psZPosTo = (static_cast<sqlite3_int64>(z) << 24) + 0x7fffff;
|
||||
sqlite3_int64 psPosZero = static_cast<sqlite3_int64>(x);
|
||||
psPosZero += static_cast<sqlite3_int64>(z) << 24;
|
||||
sqlite3_int64 psYPosFrom = 0;
|
||||
sqlite3_int64 psYPosTo = static_cast<sqlite3_int64>(y) << 12;
|
||||
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 1, psZPosFrom);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 2, psZPosTo);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 3, psPosZero);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 4, psYPosFrom);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 5, psYPosTo);
|
||||
|
||||
cacheBlocks(m_blocksYRangeStatement);
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
|
||||
{
|
||||
int result = 0;
|
||||
@ -183,9 +135,8 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(SQLstatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(SQLstatement, 1));
|
||||
int size = sqlite3_column_bytes(SQLstatement, 1);
|
||||
m_blockCache[blocknum] = DBBlock(blocknum, ustring(data, size));
|
||||
m_blockCache[blocknum] = ustring(data, size);
|
||||
m_blocksCachedCount++;
|
||||
//std::cerr << "Cache block " << blocknum << " from database" << std::endl;
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else {
|
||||
@ -195,36 +146,30 @@ void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
|
||||
sqlite3_reset(SQLstatement);
|
||||
}
|
||||
|
||||
DBBlock DBSQLite3::getBlockOnPos(int x, int y, int z)
|
||||
DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
|
||||
{
|
||||
sqlite3_int64 psPos;
|
||||
psPos = static_cast<sqlite3_int64>(x);
|
||||
psPos += static_cast<sqlite3_int64>(y) << 12;
|
||||
psPos += static_cast<sqlite3_int64>(z) << 24;
|
||||
//std::cerr << "Block " << x << "," << y << "," << z << " -> " << psPos << std::endl;
|
||||
|
||||
m_blocksReadCount++;
|
||||
|
||||
BlockCache::const_iterator DBBlockSearch;
|
||||
DBBlockSearch = m_blockCache.find(psPos);
|
||||
DBBlockSearch = m_blockCache.find(pos.databasePos());
|
||||
if (DBBlockSearch == m_blockCache.end()) {
|
||||
if (cacheWorldRow) {
|
||||
m_blockCache.clear();
|
||||
cacheBlocksOnZRaw(z);
|
||||
DBBlockSearch = m_blockCache.find(psPos);
|
||||
cacheBlocksOnZRaw(pos.z);
|
||||
DBBlockSearch = m_blockCache.find(pos.databasePos());
|
||||
if (DBBlockSearch != m_blockCache.end()) {
|
||||
return DBBlockSearch->second;
|
||||
return Block(pos, DBBlockSearch->second);
|
||||
}
|
||||
else {
|
||||
return DBBlock(0, (const unsigned char *)"");
|
||||
return Block(pos,reinterpret_cast<const unsigned char *>(""));
|
||||
}
|
||||
}
|
||||
else {
|
||||
return getBlockOnPosRaw(psPos);
|
||||
return getBlockOnPosRaw(pos);
|
||||
}
|
||||
}
|
||||
else {
|
||||
return DBBlockSearch->second;
|
||||
return Block(pos, DBBlockSearch->second);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
20
db-sqlite3.h
20
db-sqlite3.h
@ -3,7 +3,7 @@
|
||||
|
||||
#include "db.h"
|
||||
#include <sqlite3.h>
|
||||
#if _cplusplus == 201103L
|
||||
#if _cplusplus >= 201103L
|
||||
#include <unordered_map>
|
||||
#else
|
||||
#include <map>
|
||||
@ -11,11 +11,13 @@
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
#include "types.h"
|
||||
|
||||
class DBSQLite3 : public DB {
|
||||
#if _cplusplus == 201103L
|
||||
typedef std::unordered_map<uint64_t, DBBlock> BlockCache;
|
||||
#if _cplusplus >= 201103L
|
||||
typedef std::unordered_map<int64_t, ustring> BlockCache;
|
||||
#else
|
||||
typedef std::map<uint64_t, DBBlock> BlockCache;
|
||||
typedef std::map<int64_t, ustring> BlockCache;
|
||||
#endif
|
||||
public:
|
||||
bool cacheWorldRow;
|
||||
@ -23,8 +25,8 @@ public:
|
||||
virtual int getBlocksUnCachedCount(void);
|
||||
virtual int getBlocksCachedCount(void);
|
||||
virtual int getBlocksReadCount(void);
|
||||
virtual std::vector<int64_t> getBlockPos();
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z);
|
||||
virtual const BlockPosList &getBlockPos();
|
||||
virtual Block getBlockOnPos(const BlockPos &pos);
|
||||
~DBSQLite3();
|
||||
private:
|
||||
int m_blocksReadCount;
|
||||
@ -34,16 +36,14 @@ private:
|
||||
sqlite3_stmt *m_blockPosListStatement;
|
||||
sqlite3_stmt *m_blocksOnZStatement;
|
||||
sqlite3_stmt *m_blockOnPosStatement;
|
||||
sqlite3_stmt *m_blocksYRangeStatement;
|
||||
std::ostringstream m_getBlockSetStatementBlocks;
|
||||
BlockCache m_blockCache;
|
||||
BlockPosList m_BlockPosList;
|
||||
|
||||
void prepareBlocksOnZStatement(void);
|
||||
void prepareBlockOnPosStatement(void);
|
||||
void prepareBlocksYRangeStatement(void);
|
||||
void cacheBlocksYRangeRaw(int x, int y, int z);
|
||||
void cacheBlocksOnZRaw(int zPos);
|
||||
DBBlock getBlockOnPosRaw(sqlite3_int64 psPos);
|
||||
Block getBlockOnPosRaw(const BlockPos &pos);
|
||||
void cacheBlocks(sqlite3_stmt *SQLstatement);
|
||||
};
|
||||
|
||||
|
||||
13
db.h
13
db.h
@ -3,21 +3,22 @@
|
||||
|
||||
#include <stdint.h>
|
||||
#include <vector>
|
||||
#include <list>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
// we cannot use ... char>> here because mingw-gcc is f**king retarded (caring about whitespace and shit)
|
||||
typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
|
||||
typedef std::list<DBBlock> DBBlockList;
|
||||
#include "types.h"
|
||||
#include "BlockPos.h"
|
||||
|
||||
|
||||
class DB {
|
||||
public:
|
||||
virtual std::vector<int64_t> getBlockPos()=0;
|
||||
typedef std::pair<BlockPos, ustring> Block;
|
||||
typedef std::vector<int64_t> BlockPosList;
|
||||
virtual const BlockPosList &getBlockPos()=0;
|
||||
virtual int getBlocksUnCachedCount(void)=0;
|
||||
virtual int getBlocksCachedCount(void)=0;
|
||||
virtual int getBlocksReadCount(void)=0;
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z)=0;
|
||||
virtual Block getBlockOnPos(const BlockPos &pos)=0;
|
||||
};
|
||||
|
||||
#endif // _DB_H
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user