adrido
/
minetest-mapper-cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize database access - load only blocks that are needed

master
Rogier 2014-03-24 22:43:32 +01:00
parent c566d986cf
commit ecbe59ac82
8 changed files with 212 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
TileGenerator.cpp
View File

@ -95,6 +95,7 @@ static inline int colorSafeBounds(int color)
TileGenerator::TileGenerator():
verboseCoordinates(false),
verboseStatistics(false),
m_bgColor(255, 255, 255),
m_scaleColor(0, 0, 0),
m_originColor(255, 0, 0),
@ -408,7 +409,7 @@ void TileGenerator::loadBlocks()
if (pos.z > m_zMax) {
m_zMax = pos.z;
}
m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
m_positions.push_back(pos);
}
if (verboseCoordinates) {
cout << "World Geometry: "
@ -445,7 +446,6 @@ void TileGenerator::loadBlocks()
<< ") blocks: " << map_blocks << "\n";
}
m_positions.sort();
m_positions.unique();
}
inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
@ -484,28 +484,65 @@ std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
return out;
}
TileGenerator::Block TileGenerator::getBlockOnPos(BlockPos pos)
{
int64_t iPos;
iPos = pos.x;
iPos += static_cast<int64_t>(pos.y) << 12;
iPos += static_cast<int64_t>(pos.z) << 24;
DBBlock in = m_db->getBlockOnPos(iPos);
Block out(pos,(const unsigned char *)"");
if (!in.second.empty()) {
out = Block(decodeBlockPos(in.first), in.second);
// Verify. Just to be sure...
if (pos.x != out.first.x || pos.y != out.first.y || pos.z != out.first.z) {
std::ostringstream oss;
oss << "Got unexpexted block: "
<< out.first.x << "," << out.first.y << "," << out.first.z
<< " from database. Requested: "
<< pos.x << "," << pos.y << "," << pos.z;
throw std::runtime_error(oss.str());
}
}
else {
// I can't imagine this to be possible...
std::ostringstream oss;
oss << "Failed to get block: "
<< pos.x << "," << pos.y << "," << pos.z
<< " from database.";
throw std::runtime_error(oss.str());
}
return out;
}
void TileGenerator::renderMap()
{
std::list<int> zlist = getZValueList();
for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
int zPos = *zPosition;
std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
if (position->second != zPos) {
continue;
}
int blocks_selected = 0;
int blocks_rendered = 0;
BlockPos currentPos;
currentPos.x = INT_MIN;
currentPos.y = 0;
currentPos.z = INT_MIN;
bool allReaded = false;
for (std::list<BlockPos>::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
const BlockPos &pos = *position;
if (currentPos.x != pos.x || currentPos.z != pos.z) {
if (currentPos.z != pos.z && currentPos.z != INT_MIN && m_shading)
renderShading(currentPos.z);
for (int i = 0; i < 16; ++i) {
m_readedPixels[i] = 0;
}
int xPos = position->first;
blocks[xPos].sort();
const BlockList &blockStack = blocks[xPos];
for (BlockList::const_iterator it = blockStack.begin(); it != blockStack.end(); ++it) {
const BlockPos &pos = it->first;
const unsigned char *data = it->second.c_str();
size_t length = it->second.length();
currentPos = pos;
}
else if (allReaded) {
continue;
}
Block block = getBlockOnPos(pos);
blocks_selected++;
if (!block.second.empty()) {
const unsigned char *data = block.second.c_str();
size_t length = block.second.length();
uint8_t version = data[0];
//uint8_t flags = data[1];
@ -586,21 +623,20 @@ void TileGenerator::renderMap()
}
renderMapBlock(mapData, pos, version);
blocks_rendered++;
bool allReaded = true;
allReaded = true;
for (int i = 0; i < 16; ++i) {
if (m_readedPixels[i] != 0xffff) {
allReaded = false;
}
}
if (allReaded) {
break;
}
}
}
if(m_shading)
renderShading(zPos);
}
if(currentPos.z != INT_MIN && m_shading)
renderShading(currentPos.z);
if (verboseStatistics)
cout << "Statistics: Blocks selected: " << blocks_selected << "; blocks rendered: " << blocks_rendered << std::endl;
}
inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version)
@ -729,8 +765,8 @@ void TileGenerator::renderPlayers(const std::string &inputPath)
inline std::list<int> TileGenerator::getZValueList() const
{
std::list<int> zlist;
for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
zlist.push_back(position->second);
for (std::list<BlockPos>::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
zlist.push_back(position->z);
}
zlist.sort();
zlist.unique();

34
TileGenerator.h
View File

@ -32,6 +32,11 @@ struct BlockPos {
int x;
int y;
int z;
// operator< should order the positions in the
// order the corresponding pixels are generated:
// First (most significant): z coordinate, descending (i.e. reversed)
// Then : x coordinate, ascending
// Last (least significant): y coordinate, descending (i.e. reversed)
bool operator<(const BlockPos& p) const
{
if (z > p.z) {
@ -40,20 +45,33 @@ struct BlockPos {
if (z < p.z) {
return false;
}
if (x < p.x) {
return true;
}
if (x > p.x) {
return false;
}
if (y > p.y) {
return true;
}
if (y < p.y) {
return false;
}
if (x > p.x) {
return false;
}
bool operator==(const BlockPos& p) const
{
if (z != p.z) {
return false;
}
if (y != p.y) {
return false;
}
if (x != p.x) {
return false;
}
return true;
}
if (x < p.x) {
return false;
}
return false;
}
};
@ -92,6 +110,7 @@ private:
void createImage();
void renderMap();
std::list<int> getZValueList() const;
Block getBlockOnPos(BlockPos pos);
std::map<int, BlockList> getBlocksOnZ(int zPos);
void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
void renderShading(int zPos);
@ -105,6 +124,7 @@ private:
public:
bool verboseCoordinates;
bool verboseStatistics;
private:
Color m_bgColor;
@ -138,7 +158,7 @@ private:
int m_reqYMaxNode; // Node offset within a map block
int m_mapWidth;
int m_mapHeight;
std::list<std::pair<int, int> > m_positions;
std::list<BlockPos> m_positions;
std::map<int, std::string> m_nameMap;
ColorMap m_colors;
uint16_t m_readedPixels[16];

13
db-leveldb.cpp
View File

@ -62,3 +62,16 @@ DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
return blocks;
}
DBBlock DBLevelDB::getBlocksOnPos(int64_t iPos)
{
DBBlock block(0,(const unsigned char *)"");
std::string datastr;
leveldb::Status status;
status = m_db->Get(leveldb::ReadOptions(), i64tos(Pos), &datastr);
if(status.ok())
block = DBBlock( iPos, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) );
return block;
}

1
db-leveldb.h
View File

@ -10,6 +10,7 @@ public:
DBLevelDB(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
virtual DBBlock getBlockOnPos(int64_t iPos);
~DBLevelDB();
private:
leveldb::DB *m_db;

31
db-sqlite3.cpp
View File

@ -77,3 +77,34 @@ DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
return blocks;
}
DBBlock DBSQLite3::getBlockOnPos(int64_t iPos)
{
std::string sql = "SELECT pos, data FROM blocks WHERE pos == ?";
if (!m_getBlocksOnPosStatement && sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &m_getBlocksOnPosStatement, 0) != SQLITE_OK) {
throw std::runtime_error("Failed to prepare statement");
}
DBBlock block(0,(const unsigned char *)"");
sqlite3_int64 psPos = static_cast<sqlite3_int64>(iPos);
sqlite3_bind_int64(m_getBlocksOnPosStatement, 1, psPos);
int result = 0;
while (true) {
result = sqlite3_step(m_getBlocksOnPosStatement);
if(result == SQLITE_ROW) {
sqlite3_int64 blocknum = sqlite3_column_int64(m_getBlocksOnPosStatement, 0);
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(m_getBlocksOnPosStatement, 1));
int size = sqlite3_column_bytes(m_getBlocksOnPosStatement, 1);
block = DBBlock(blocknum, std::basic_string<unsigned char>(data, size));
break;
} else if (result == SQLITE_BUSY) { // Wait some time and try again
usleep(10000);
} else {
break;
}
}
sqlite3_reset(m_getBlocksOnPosStatement);
return block;
}

2
db-sqlite3.h
View File

@ -3,12 +3,14 @@
#include "db.h"
#include <sqlite3.h>
#include <map>
class DBSQLite3 : public DB {
public:
DBSQLite3(const std::string &mapdir);
virtual std::vector<int64_t> getBlockPos();
virtual DBBlockList getBlocksOnZ(int zPos);
virtual DBBlock getBlockOnPos(int64_t iPos);
~DBSQLite3();
private:
sqlite3 *m_db;

1
db.h
View File

@ -15,6 +15,7 @@ class DB {
public:
virtual std::vector<int64_t> getBlockPos()=0;
virtual DBBlockList getBlocksOnZ(int zPos)=0;
virtual DBBlock getBlockOnPos(int64_t iPos)=0;
};
#endif // _DB_H

1
mapper.cpp
View File

@ -114,6 +114,7 @@ int main(int argc, char *argv[])
break;
case 'v':
generator.verboseCoordinates = true;
generator.verboseStatistics = true;
break;
case 'H':
generator.setShading(false);