Database performance tweaks
Added the option to bulk load and cache entire world rows at a time when using sqlite. Caching is implemented in the database class this time around. Enabled using --sqlite-cacheworldrow on the command line. This is essentially re-introduces the database access strategy that was used originally, but as an option. Currently, one block is read at a time from the database. The original behavior was to read and cache one entire world row at a time, irrespective of whether or not only a (small) subsection of the world was being mapped. Under some circumstances, in particular if the entire world is mapped or a relatively large percentage of it (blocks-wise), the bulk loading and caching may outperform the one-by-one strategy, even though it usually loads many more blocks than are actually needed. It seems that leveldb won't benefit from any kind of caching or bulk loading of blocks, so none was implemented. Leveldb does compile now. Still not tested on a database :-( Improved the database statistics (printed with --verbose).master
Rogier
parent
4ca967d40d
commit
10e6a746cd
|
|
@ -95,6 +95,11 @@ geometry:
|
|||
forcegeometry:
|
||||
Generate a map of the requested size, even if the world is smaller.
|
||||
|
||||
sqlite-cacheworldrow:
|
||||
When using sqlite, read an entire world row at one, instead of reading
|
||||
one block at a time.
|
||||
This may improve performance when a large percentage of the world is mapped.
|
||||
|
||||
verbose:
|
||||
report some useful/ interesting information:
|
||||
- maximum coordinates of the world
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ TileGenerator::TileGenerator():
|
|||
m_border(0),
|
||||
m_backend("sqlite3"),
|
||||
m_forceGeom(false),
|
||||
m_sqliteCacheWorldRow(false),
|
||||
m_image(0),
|
||||
m_xMin(INT_MAX/16-1),
|
||||
m_xMax(INT_MIN/16+1),
|
||||
|
|
@ -142,6 +143,11 @@ void TileGenerator::setForceGeom(bool forceGeom)
|
|||
m_forceGeom = forceGeom;
|
||||
}
|
||||
|
||||
void TileGenerator::setSqliteCacheWorldRow(bool cacheWorldRow)
|
||||
{
|
||||
m_sqliteCacheWorldRow = cacheWorldRow;
|
||||
}
|
||||
|
||||
void TileGenerator::setScaleColor(const std::string &scaleColor)
|
||||
{
|
||||
m_scaleColor = parseColor(scaleColor);
|
||||
|
|
@ -319,8 +325,11 @@ void TileGenerator::parseColorsStream(std::istream &in)
|
|||
|
||||
void TileGenerator::openDb(const std::string &input)
|
||||
{
|
||||
if(m_backend == "sqlite3")
|
||||
m_db = new DBSQLite3(input);
|
||||
if(m_backend == "sqlite3") {
|
||||
DBSQLite3 *db;
|
||||
m_db = db = new DBSQLite3(input);
|
||||
db->cacheWorldRow = m_sqliteCacheWorldRow;
|
||||
}
|
||||
#if USE_LEVELDB
|
||||
else if(m_backend == "leveldb")
|
||||
m_db = new DBLevelDB(input);
|
||||
|
|
@ -469,28 +478,9 @@ void TileGenerator::createImage()
|
|||
gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, rgb2int(m_bgColor.r, m_bgColor.g, m_bgColor.b));
|
||||
}
|
||||
|
||||
std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
|
||||
{
|
||||
DBBlockList in = m_db->getBlocksOnZ(zPos);
|
||||
std::map<int, BlockList> out;
|
||||
for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
|
||||
Block b = Block(decodeBlockPos(it->first), it->second);
|
||||
if(out.find(b.first.x) == out.end()) {
|
||||
BlockList bl;
|
||||
out[b.first.x] = bl;
|
||||
}
|
||||
out[b.first.x].push_back(b);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
|
||||
{
|
||||
int64_t iPos;
|
||||
iPos = pos.x;
|
||||
iPos += static_cast<int64_t>(pos.y) << 12;
|
||||
iPos += static_cast<int64_t>(pos.z) << 24;
|
||||
DBBlock in = m_db->getBlockOnPos(iPos);
|
||||
DBBlock in = m_db->getBlockOnPos(pos.x, pos.y, pos.z);
|
||||
Block out(pos,(const unsigned char *)"");
|
||||
|
||||
if (!in.second.empty()) {
|
||||
|
|
@ -518,7 +508,6 @@ TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
|
|||
|
||||
void TileGenerator::renderMap()
|
||||
{
|
||||
int blocks_selected = 0;
|
||||
int blocks_rendered = 0;
|
||||
BlockPos currentPos;
|
||||
currentPos.x = INT_MIN;
|
||||
|
|
@ -539,7 +528,6 @@ void TileGenerator::renderMap()
|
|||
continue;
|
||||
}
|
||||
Block block = getBlockOnPos(pos);
|
||||
blocks_selected++;
|
||||
if (!block.second.empty()) {
|
||||
const unsigned char *data = block.second.c_str();
|
||||
size_t length = block.second.length();
|
||||
|
|
@ -636,7 +624,12 @@ void TileGenerator::renderMap()
|
|||
if(currentPos.z != INT_MIN && m_shading)
|
||||
renderShading(currentPos.z);
|
||||
if (verboseStatistics)
|
||||
cout << "Statistics: Blocks selected: " << blocks_selected << "; blocks rendered: " << blocks_rendered << std::endl;
|
||||
cout << "Statistics"
|
||||
<< ": blocks read: " << m_db->getBlocksReadCount()
|
||||
<< "; (" << m_db->getBlocksCachedCount() << " cached + "
|
||||
<< m_db->getBlocksUnCachedCount() << " uncached)"
|
||||
<< "; blocks rendered: " << blocks_rendered
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ public:
|
|||
void setMinY(int y);
|
||||
void setMaxY(int y);
|
||||
void setForceGeom(bool forceGeom);
|
||||
void setSqliteCacheWorldRow(bool cacheWorldRow);
|
||||
void parseColorsFile(const std::string &fileName);
|
||||
void setBackend(std::string backend);
|
||||
void generate(const std::string &input, const std::string &output);
|
||||
|
|
@ -111,7 +112,6 @@ private:
|
|||
void renderMap();
|
||||
std::list<int> getZValueList() const;
|
||||
Block getBlockOnPos(BlockPos pos);
|
||||
std::map<int, BlockList> getBlocksOnZ(int zPos);
|
||||
void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
|
||||
void renderShading(int zPos);
|
||||
void renderScale();
|
||||
|
|
@ -138,6 +138,7 @@ private:
|
|||
int m_border;
|
||||
std::string m_backend;
|
||||
bool m_forceGeom;
|
||||
bool m_sqliteCacheWorldRow;
|
||||
|
||||
DB *m_db;
|
||||
gdImagePtr m_image;
|
||||
|
|
|
|||
|
|
@ -15,7 +15,11 @@ inline std::string i64tos(int64_t i) {
|
|||
return o.str();
|
||||
}
|
||||
|
||||
DBLevelDB::DBLevelDB(const std::string &mapdir) {
|
||||
DBLevelDB::DBLevelDB(const std::string &mapdir) :
|
||||
m_blocksReadCount(0),
|
||||
m_blocksCachedCount(0),
|
||||
m_blocksUnCachedCount(0)
|
||||
{
|
||||
leveldb::Options options;
|
||||
options.create_if_missing = false;
|
||||
leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db);
|
||||
|
|
@ -27,6 +31,21 @@ DBLevelDB::~DBLevelDB() {
|
|||
delete m_db;
|
||||
}
|
||||
|
||||
int DBLevelDB::getBlocksReadCount(void)
|
||||
{
|
||||
return m_blocksReadCount;
|
||||
}
|
||||
|
||||
int DBLevelDB::getBlocksCachedCount(void)
|
||||
{
|
||||
return m_blocksCachedCount;
|
||||
}
|
||||
|
||||
int DBLevelDB::getBlocksUnCachedCount(void)
|
||||
{
|
||||
return m_blocksUnCachedCount;
|
||||
}
|
||||
|
||||
std::vector<int64_t> DBLevelDB::getBlockPos() {
|
||||
std::vector<int64_t> vec;
|
||||
std::set<int64_t> s;
|
||||
|
|
@ -40,37 +59,23 @@ std::vector<int64_t> DBLevelDB::getBlockPos() {
|
|||
return vec;
|
||||
}
|
||||
|
||||
DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
|
||||
{
|
||||
DBBlockList blocks;
|
||||
std::string datastr;
|
||||
leveldb::Status status;
|
||||
|
||||
int64_t psMin;
|
||||
int64_t psMax;
|
||||
psMin = (zPos * 16777216l) - 0x800000;
|
||||
psMax = (zPos * 16777216l) + 0x7fffff;
|
||||
|
||||
for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache)
|
||||
if(m_bpcache.find(i) == m_bpcache.end())
|
||||
continue;
|
||||
status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
|
||||
if(status.ok())
|
||||
blocks.push_back( DBBlock( i, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) ) );
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
DBBlock DBLevelDB::getBlocksOnPos(int64_t iPos)
|
||||
DBBlock DBLevelDB::getBlockOnPos(int x, int y, int z)
|
||||
{
|
||||
int64_t iPos;
|
||||
DBBlock block(0,(const unsigned char *)"");
|
||||
std::string datastr;
|
||||
leveldb::Status status;
|
||||
|
||||
status = m_db->Get(leveldb::ReadOptions(), i64tos(Pos), &datastr);
|
||||
if(status.ok())
|
||||
iPos = static_cast<int64_t>(x);
|
||||
iPos += static_cast<int64_t>(y) << 12;
|
||||
iPos += static_cast<int64_t>(z) << 24;
|
||||
|
||||
status = m_db->Get(leveldb::ReadOptions(), i64tos(iPos), &datastr);
|
||||
if(status.ok()) {
|
||||
block = DBBlock( iPos, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) );
|
||||
m_blocksReadCount++;
|
||||
m_blocksUnCachedCount++;
|
||||
}
|
||||
|
||||
return block;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,16 @@
|
|||
class DBLevelDB : public DB {
|
||||
public:
|
||||
DBLevelDB(const std::string &mapdir);
|
||||
virtual int getBlocksUnCachedCount(void);
|
||||
virtual int getBlocksCachedCount(void);
|
||||
virtual int getBlocksReadCount(void);
|
||||
virtual std::vector<int64_t> getBlockPos();
|
||||
virtual DBBlockList getBlocksOnZ(int zPos);
|
||||
virtual DBBlock getBlockOnPos(int64_t iPos);
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z);
|
||||
~DBLevelDB();
|
||||
private:
|
||||
int m_blocksReadCount;
|
||||
int m_blocksCachedCount;
|
||||
int m_blocksUnCachedCount;
|
||||
leveldb::DB *m_db;
|
||||
std::set<int64_t> m_bpcache;
|
||||
};
|
||||
|
|
|
|||
209
db-sqlite3.cpp
209
db-sqlite3.cpp
|
|
@ -2,10 +2,15 @@
|
|||
#include <stdexcept>
|
||||
#include <unistd.h> // for usleep
|
||||
|
||||
|
||||
DBSQLite3::DBSQLite3(const std::string &mapdir) :
|
||||
m_getBlockPosStatement(NULL),
|
||||
m_getBlocksOnZStatement(NULL),
|
||||
m_getBlocksOnPosStatement(NULL)
|
||||
cacheWorldRow(false),
|
||||
m_blocksReadCount(0),
|
||||
m_blocksCachedCount(0),
|
||||
m_blocksUnCachedCount(0),
|
||||
m_blockPosListStatement(NULL),
|
||||
m_blocksOnZStatement(NULL),
|
||||
m_blockOnPosStatement(NULL)
|
||||
{
|
||||
|
||||
std::string db_name = mapdir + "map.sqlite";
|
||||
|
|
@ -15,21 +20,36 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) :
|
|||
}
|
||||
|
||||
DBSQLite3::~DBSQLite3() {
|
||||
if (m_getBlockPosStatement) sqlite3_finalize(m_getBlockPosStatement);
|
||||
if (m_getBlocksOnZStatement) sqlite3_finalize(m_getBlocksOnZStatement);
|
||||
if (m_getBlocksOnPosStatement) sqlite3_finalize(m_getBlocksOnPosStatement);
|
||||
if (m_blockPosListStatement) sqlite3_finalize(m_blockPosListStatement);
|
||||
if (m_blocksOnZStatement) sqlite3_finalize(m_blocksOnZStatement);
|
||||
if (m_blockOnPosStatement) sqlite3_finalize(m_blockOnPosStatement);
|
||||
sqlite3_close(m_db);
|
||||
}
|
||||
|
||||
int DBSQLite3::getBlocksReadCount(void)
|
||||
{
|
||||
return m_blocksReadCount;
|
||||
}
|
||||
|
||||
int DBSQLite3::getBlocksCachedCount(void)
|
||||
{
|
||||
return m_blocksCachedCount;
|
||||
}
|
||||
|
||||
int DBSQLite3::getBlocksUnCachedCount(void)
|
||||
{
|
||||
return m_blocksUnCachedCount;
|
||||
}
|
||||
|
||||
std::vector<int64_t> DBSQLite3::getBlockPos() {
|
||||
std::vector<int64_t> vec;
|
||||
std::string sql = "SELECT pos FROM blocks";
|
||||
if (m_getBlockPosStatement || sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_getBlockPosStatement, 0) == SQLITE_OK) {
|
||||
if (m_blockPosListStatement || sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blockPosListStatement, 0) == SQLITE_OK) {
|
||||
int result = 0;
|
||||
while (true) {
|
||||
result = sqlite3_step(m_getBlockPosStatement);
|
||||
result = sqlite3_step(m_blockPosListStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_getBlockPosStatement, 0);
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
|
||||
vec.push_back(blocknum);
|
||||
} else if (result == SQLITE_BUSY) // Wait some time and try again
|
||||
usleep(10000);
|
||||
|
|
@ -37,17 +57,58 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
|
|||
break;
|
||||
}
|
||||
} else {
|
||||
sqlite3_reset(m_blockPosListStatement);
|
||||
throw std::runtime_error("Failed to get list of MapBlocks");
|
||||
}
|
||||
sqlite3_reset(m_blockPosListStatement);
|
||||
return vec;
|
||||
}
|
||||
|
||||
DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
|
||||
void DBSQLite3::prepareBlocksOnZStatement(void)
|
||||
{
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
|
||||
if (!m_getBlocksOnZStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_getBlocksOnZStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare statement");
|
||||
//std::string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ? AND ?)";
|
||||
if (!m_blocksOnZStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blocksOnZStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare statement (blocksOnZStatement)");
|
||||
}
|
||||
}
|
||||
|
||||
void DBSQLite3::prepareBlockOnPosStatement(void)
|
||||
{
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE pos == ?";
|
||||
if (!m_blockOnPosStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blockOnPosStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)");
|
||||
}
|
||||
}
|
||||
|
||||
// Apparently, this attempt at being smart, is actually quite inefficient for sqlite.
|
||||
//
|
||||
// For larger subsections of the map, it performs much worse than caching an entire
|
||||
// world row (i.e. z coordinate). In cases where it *is* more efficient, no caching is
|
||||
// much more efficient still.
|
||||
//
|
||||
// It seems that any computation on pos severely affects the performance (?)...
|
||||
//
|
||||
// For the moment, this function is not used.
|
||||
void DBSQLite3::prepareBlocksYRangeStatement(void)
|
||||
{
|
||||
// This one seems to perform best:
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3)&4095 == 0 AND (pos-?3 BETWEEN ?4 AND ?5)";
|
||||
// These perform worse:
|
||||
//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) AND (pos-?3 BETWEEN ?4 AND ?5)";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos-?3 BETWEEN ?4 AND ?5) AND (pos-?3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5) AND (pos3)&4095 == 0";
|
||||
//std::string sql = "SELECT pos, data FROM (select pos, (pos-?3) AS pos3, data FROM blocks WHERE (pos BETWEEN ?1 AND ?2) ) WHERE (pos3 BETWEEN ?4 AND ?5)";
|
||||
if (!m_blocksYRangeStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_blocksYRangeStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blocksYRangeStatement)");
|
||||
}
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocksOnZRaw(int zPos)
|
||||
{
|
||||
prepareBlocksOnZStatement();
|
||||
|
||||
DBBlockList blocks;
|
||||
|
||||
sqlite3_int64 psMin;
|
||||
|
|
@ -55,47 +116,31 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
|
|||
|
||||
psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
|
||||
psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
|
||||
sqlite3_bind_int64(m_getBlocksOnZStatement, 1, psMin);
|
||||
sqlite3_bind_int64(m_getBlocksOnZStatement, 2, psMax);
|
||||
|
||||
int result = 0;
|
||||
while (true) {
|
||||
result = sqlite3_step(m_getBlocksOnZStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_getBlocksOnZStatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_getBlocksOnZStatement, 1));
|
||||
int size = sqlite3_column_bytes(m_getBlocksOnZStatement, 1);
|
||||
blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size)));
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_reset(m_getBlocksOnZStatement);
|
||||
sqlite3_bind_int64(m_blocksOnZStatement, 1, psMin);
|
||||
sqlite3_bind_int64(m_blocksOnZStatement, 2, psMax);
|
||||
|
||||
return blocks;
|
||||
cacheBlocks(m_blocksOnZStatement);
|
||||
}
|
||||
|
||||
DBBlock DBSQLite3::getBlockOnPos(int64_t iPos)
|
||||
DBBlock DBSQLite3::getBlockOnPosRaw(sqlite3_int64 psPos)
|
||||
{
|
||||
std::string sql = "SELECT pos, data FROM blocks WHERE pos == ?";
|
||||
if (!m_getBlocksOnPosStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_getBlocksOnPosStatement, 0) != SQLITE_OK) {
|
||||
throw std::runtime_error("Failed to prepare statement");
|
||||
}
|
||||
prepareBlockOnPosStatement();
|
||||
|
||||
DBBlock block(0,(const unsigned char *)"");
|
||||
|
||||
sqlite3_int64 psPos = static_cast<sqlite3_int64>(iPos);
|
||||
sqlite3_bind_int64(m_getBlocksOnPosStatement, 1, psPos);
|
||||
|
||||
int result = 0;
|
||||
|
||||
sqlite3_bind_int64(m_blockOnPosStatement, 1, psPos);
|
||||
|
||||
while (true) {
|
||||
result = sqlite3_step(m_getBlocksOnPosStatement);
|
||||
result = sqlite3_step(m_blockOnPosStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_getBlocksOnPosStatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_getBlocksOnPosStatement, 1));
|
||||
int size = sqlite3_column_bytes(m_getBlocksOnPosStatement, 1);
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockOnPosStatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_blockOnPosStatement, 1));
|
||||
int size = sqlite3_column_bytes(m_blockOnPosStatement, 1);
|
||||
block = DBBlock(blocknum, std::basic_string<unsigned char>(data, size));
|
||||
m_blocksUnCachedCount++;
|
||||
//std::cerr << "Read block " << blocknum << " from database" << std::endl;
|
||||
break;
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
|
|
@ -103,8 +148,82 @@ DBBlock DBSQLite3::getBlockOnPos(int64_t iPos)
|
|||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_reset(m_getBlocksOnPosStatement);
|
||||
sqlite3_reset(m_blockOnPosStatement);
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocksYRangeRaw(int x, int y, int z)
|
||||
{
|
||||
prepareBlocksYRangeStatement();
|
||||
|
||||
sqlite3_int64 psZPosFrom = (static_cast<sqlite3_int64>(z) << 24) - 0x800000;
|
||||
sqlite3_int64 psZPosTo = (static_cast<sqlite3_int64>(z) << 24) + 0x7fffff;
|
||||
sqlite3_int64 psPosZero = static_cast<sqlite3_int64>(x);
|
||||
psPosZero += static_cast<sqlite3_int64>(z) << 24;
|
||||
sqlite3_int64 psYPosFrom = 0;
|
||||
sqlite3_int64 psYPosTo = static_cast<sqlite3_int64>(y) << 12;
|
||||
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 1, psZPosFrom);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 2, psZPosTo);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 3, psPosZero);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 4, psYPosFrom);
|
||||
sqlite3_bind_int64(m_blocksYRangeStatement, 5, psYPosTo);
|
||||
|
||||
cacheBlocks(m_blocksYRangeStatement);
|
||||
}
|
||||
|
||||
void DBSQLite3::cacheBlocks(sqlite3_stmt *SQLstatement)
|
||||
{
|
||||
int result = 0;
|
||||
while (true) {
|
||||
result = sqlite3_step(SQLstatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(SQLstatement, 0);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(SQLstatement, 1));
|
||||
int size = sqlite3_column_bytes(SQLstatement, 1);
|
||||
m_blockCache[blocknum] = DBBlock(blocknum, std::basic_string<unsigned char>(data, size));
|
||||
m_blocksCachedCount++;
|
||||
//std::cerr << "Cache block " << blocknum << " from database" << std::endl;
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_reset(SQLstatement);
|
||||
}
|
||||
|
||||
DBBlock DBSQLite3::getBlockOnPos(int x, int y, int z)
|
||||
{
|
||||
sqlite3_int64 psPos;
|
||||
psPos = static_cast<sqlite3_int64>(x);
|
||||
psPos += static_cast<sqlite3_int64>(y) << 12;
|
||||
psPos += static_cast<sqlite3_int64>(z) << 24;
|
||||
//std::cerr << "Block " << x << "," << y << "," << z << " -> " << psPos << std::endl;
|
||||
|
||||
m_blocksReadCount++;
|
||||
|
||||
BlockCache::const_iterator DBBlockSearch;
|
||||
DBBlockSearch = m_blockCache.find(psPos);
|
||||
if (DBBlockSearch == m_blockCache.end()) {
|
||||
if (cacheWorldRow) {
|
||||
m_blockCache.clear();
|
||||
cacheBlocksOnZRaw(z);
|
||||
DBBlockSearch = m_blockCache.find(psPos);
|
||||
if (DBBlockSearch != m_blockCache.end()) {
|
||||
return DBBlockSearch->second;
|
||||
}
|
||||
else {
|
||||
return DBBlock(0, (const unsigned char *)"");
|
||||
}
|
||||
}
|
||||
else {
|
||||
return getBlockOnPosRaw(psPos);
|
||||
}
|
||||
}
|
||||
else {
|
||||
return DBBlockSearch->second;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
38
db-sqlite3.h
38
db-sqlite3.h
|
|
@ -3,20 +3,48 @@
|
|||
|
||||
#include "db.h"
|
||||
#include <sqlite3.h>
|
||||
#if _cplusplus == 201103L
|
||||
#include <unordered_map>
|
||||
#else
|
||||
#include <map>
|
||||
#endif
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
class DBSQLite3 : public DB {
|
||||
#if _cplusplus == 201103L
|
||||
typedef std::unordered_map<uint64_t, DBBlock> BlockCache;
|
||||
#else
|
||||
typedef std::map<uint64_t, DBBlock> BlockCache;
|
||||
#endif
|
||||
public:
|
||||
bool cacheWorldRow;
|
||||
DBSQLite3(const std::string &mapdir);
|
||||
virtual int getBlocksUnCachedCount(void);
|
||||
virtual int getBlocksCachedCount(void);
|
||||
virtual int getBlocksReadCount(void);
|
||||
virtual std::vector<int64_t> getBlockPos();
|
||||
virtual DBBlockList getBlocksOnZ(int zPos);
|
||||
virtual DBBlock getBlockOnPos(int64_t iPos);
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z);
|
||||
~DBSQLite3();
|
||||
private:
|
||||
int m_blocksReadCount;
|
||||
int m_blocksCachedCount;
|
||||
int m_blocksUnCachedCount;
|
||||
sqlite3 *m_db;
|
||||
sqlite3_stmt *m_getBlockPosStatement;
|
||||
sqlite3_stmt *m_getBlocksOnZStatement;
|
||||
sqlite3_stmt *m_getBlocksOnPosStatement;
|
||||
sqlite3_stmt *m_blockPosListStatement;
|
||||
sqlite3_stmt *m_blocksOnZStatement;
|
||||
sqlite3_stmt *m_blockOnPosStatement;
|
||||
sqlite3_stmt *m_blocksYRangeStatement;
|
||||
std::ostringstream m_getBlockSetStatementBlocks;
|
||||
BlockCache m_blockCache;
|
||||
|
||||
void prepareBlocksOnZStatement(void);
|
||||
void prepareBlockOnPosStatement(void);
|
||||
void prepareBlocksYRangeStatement(void);
|
||||
void cacheBlocksYRangeRaw(int x, int y, int z);
|
||||
void cacheBlocksOnZRaw(int zPos);
|
||||
DBBlock getBlockOnPosRaw(sqlite3_int64 psPos);
|
||||
void cacheBlocks(sqlite3_stmt *SQLstatement);
|
||||
};
|
||||
|
||||
#endif // _DB_SQLITE3_H
|
||||
|
|
|
|||
6
db.h
6
db.h
|
|
@ -14,8 +14,10 @@ typedef std::list<DBBlock> DBBlockList;
|
|||
class DB {
|
||||
public:
|
||||
virtual std::vector<int64_t> getBlockPos()=0;
|
||||
virtual DBBlockList getBlocksOnZ(int zPos)=0;
|
||||
virtual DBBlock getBlockOnPos(int64_t iPos)=0;
|
||||
virtual int getBlocksUnCachedCount(void)=0;
|
||||
virtual int getBlocksCachedCount(void)=0;
|
||||
virtual int getBlocksReadCount(void)=0;
|
||||
virtual DBBlock getBlockOnPos(int x, int y, int z)=0;
|
||||
};
|
||||
|
||||
#endif // _DB_H
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@
|
|||
|
||||
using namespace std;
|
||||
|
||||
#define OPT_SQLITE_CACHEWORLDROW 0x81
|
||||
|
||||
void usage()
|
||||
{
|
||||
const char *usage_text = "minetestmapper [options]\n"
|
||||
|
|
@ -36,6 +38,7 @@ void usage()
|
|||
" --backend <sqlite3/leveldb>\n"
|
||||
" --geometry x:y+w+h\n"
|
||||
" --forcegeometry\n"
|
||||
" --sqlite-cacheworldrow\n"
|
||||
" --verbose\n"
|
||||
"Color format: '#000000'\n";
|
||||
std::cout << usage_text;
|
||||
|
|
@ -61,6 +64,7 @@ int main(int argc, char *argv[])
|
|||
{"min-y", required_argument, 0, 'a'},
|
||||
{"max-y", required_argument, 0, 'c'},
|
||||
{"backend", required_argument, 0, 'd'},
|
||||
{"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW},
|
||||
{"verbose", no_argument, 0, 'v'},
|
||||
};
|
||||
|
||||
|
|
@ -119,6 +123,9 @@ int main(int argc, char *argv[])
|
|||
case 'H':
|
||||
generator.setShading(false);
|
||||
break;
|
||||
case OPT_SQLITE_CACHEWORLDROW:
|
||||
generator.setSqliteCacheWorldRow(true);
|
||||
break;
|
||||
case 'a': {
|
||||
istringstream iss;
|
||||
iss.str(optarg);
|
||||
|
|
|
|||
Loading…
Reference in New Issue