adrido/minetest-mapper-cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modernize db-sqlite3

Browse Source
This commit is contained in:
Unknown 2018-04-11 19:16:55 +02:00
parent 467e7cea54
commit 685b136563
2 changed files with 80 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
Minetestmapper/db-sqlite3.cpp
View File

@ -2,12 +2,12 @@
#ifdef USE_SQLITE3 #ifdef USE_SQLITE3
#include <stdexcept> #include <chrono>
#include <iomanip>
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
#include <iomanip> #include <stdexcept>
#include <ctime> #include <thread>
#include "porting.h"
#define DATAVERSION_STATEMENT "PRAGMA data_version" #define DATAVERSION_STATEMENT "PRAGMA data_version"
#define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks" #define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks"
@ -18,6 +18,8 @@
#define BLOCKLIST_QUERY_SIZE_MIN 2000 #define BLOCKLIST_QUERY_SIZE_MIN 2000
#define BLOCKLIST_QUERY_SIZE_DEFAULT 250000 #define BLOCKLIST_QUERY_SIZE_DEFAULT 250000
#define sleepMs(x) std::this_thread::sleep_for(std::chrono::milliseconds(x))
// If zero, a full block list is obtained using a single query. // If zero, a full block list is obtained using a single query.
// If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used. // If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used.
int DBSQLite3::m_blockListQuerySize = 0; int DBSQLite3::m_blockListQuerySize = 0;
@ -26,66 +28,72 @@ bool DBSQLite3::warnDatabaseLockDelay = true;
void DBSQLite3::setLimitBlockListQuerySize(int count) void DBSQLite3::setLimitBlockListQuerySize(int count)
{ {
if (m_firstDatabaseInitialized) if (m_firstDatabaseInitialized) {
throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open"); throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open");
}
if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) { if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) {
std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl; std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl;
count = BLOCKLIST_QUERY_SIZE_MIN; count = BLOCKLIST_QUERY_SIZE_MIN;
} }
if (count < 0) if (count < 0) {
m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT; m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT;
else }
else {
m_blockListQuerySize = count; m_blockListQuerySize = count;
}
} }
DBSQLite3::DBSQLite3(const std::string &mapdir) : DBSQLite3::DBSQLite3(const std::string &mapdir) :
m_blocksQueriedCount(0), m_blocksQueriedCount(0),
m_blocksReadCount(0), m_blocksReadCount(0)
m_blockPosListStatement(NULL),
m_blockOnPosStatement(NULL),
m_blockOnRowidStatement(NULL)
{ {
m_firstDatabaseInitialized = true; m_firstDatabaseInitialized = true;
std::string db_name = mapdir + "map.sqlite"; std::string db_name = mapdir + "map.sqlite";
if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) { if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, nullptr) != SQLITE_OK) {
throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name); throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
} }
if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT)-1, &m_dataVersionStatement, 0)) { if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT) - 1, &m_dataVersionStatement, nullptr)) {
throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)"); throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)");
} }
if (m_blockListQuerySize == 0) { if (m_blockListQuerySize == 0) {
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) { if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) {
throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)"); throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)");
} }
} }
else { else {
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT, if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT,
sizeof(BLOCKPOSLIST_LIMITED_STATEMENT)-1, &m_blockPosListStatement, 0)) { sizeof(BLOCKPOSLIST_LIMITED_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) {
throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))"); throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))");
} }
} }
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS)-1, &m_blockOnPosStatement, 0)) { if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS) - 1, &m_blockOnPosStatement, nullptr)) {
throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)"); throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)");
} }
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID)-1, &m_blockOnRowidStatement, 0)) { if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID) - 1, &m_blockOnRowidStatement, nullptr)) {
throw std::runtime_error("Failed to prepare SQL statement (blockOnRowidStatement)"); throw std::runtime_error("Failed to prepare SQL statement (blockOnRowidStatement)");
} }
} }
DBSQLite3::~DBSQLite3() { DBSQLite3::~DBSQLite3() {
if (m_blockPosListStatement) sqlite3_finalize(m_blockPosListStatement); if (m_blockPosListStatement) {
if (m_blockOnPosStatement) sqlite3_finalize(m_blockOnPosStatement); sqlite3_finalize(m_blockPosListStatement);
if (m_blockOnRowidStatement) sqlite3_finalize(m_blockOnRowidStatement); }
if (m_blockOnPosStatement) {
sqlite3_finalize(m_blockOnPosStatement);
}
if (m_blockOnRowidStatement) {
sqlite3_finalize(m_blockOnRowidStatement);
}
sqlite3_close(m_db); sqlite3_close(m_db);
} }
int DBSQLite3::getBlocksReadCount(void) int DBSQLite3::getBlocksReadCount()
{ {
return m_blocksReadCount; return m_blocksReadCount;
} }
int DBSQLite3::getBlocksQueriedCount(void) int DBSQLite3::getBlocksQueriedCount()
{ {
return m_blocksQueriedCount; return m_blocksQueriedCount;
} }
@ -95,13 +103,16 @@ int64_t DBSQLite3::getDataVersion()
int64_t version = 0; int64_t version = 0;
for (;;) { for (;;) {
int result = sqlite3_step(m_dataVersionStatement); int result = sqlite3_step(m_dataVersionStatement);
if(result == SQLITE_ROW) if (result == SQLITE_ROW) {
version = sqlite3_column_int64(m_dataVersionStatement, 0); version = sqlite3_column_int64(m_dataVersionStatement, 0);
else if (result == SQLITE_BUSY) // Wait some time and try again }
else if (result == SQLITE_BUSY) { // Wait some time and try again
sleepMs(10); sleepMs(10);
else }
else {
break; break;
} }
}
sqlite3_reset(m_blockPosListStatement); sqlite3_reset(m_blockPosListStatement);
return version; return version;
} }
@ -142,16 +153,21 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList()
// The enforced minimum of 1000 is probably too small to avoid skipped blocks // The enforced minimum of 1000 is probably too small to avoid skipped blocks
// if heavy world generation is going on, but then, a query size of 10000 is // if heavy world generation is going on, but then, a query size of 10000 is
// also pretty small. // also pretty small.
if (m_blockListQuerySize <= 10000) if (m_blockListQuerySize <= 10000) {
querySize = m_blockListQuerySize + 1000; querySize = m_blockListQuerySize + 1000;
else if (m_blockListQuerySize <= 100000) }
querySize = m_blockListQuerySize * 1.1; // 1000 .. 10000 else if (m_blockListQuerySize <= 100000) {
else if (m_blockListQuerySize <= 1000000) querySize = static_cast<int>(m_blockListQuerySize * 1.1f); // 1000 .. 10000
querySize = m_blockListQuerySize * 1.011 + 9000; // 10100 .. 20000 }
else if (m_blockListQuerySize <= 10000000) else if (m_blockListQuerySize <= 1000000) {
querySize = m_blockListQuerySize * 1.0022 + 18000; // 20200 .. 40000 querySize = static_cast<int>(m_blockListQuerySize * 1.011f) + 9000; // 10100 .. 20000
else // More than 10M blocks per query. Most worlds are smaller than this... }
querySize = m_blockListQuerySize * 1.001 + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.) else if (m_blockListQuerySize <= 10000000) {
querySize = static_cast<int>(m_blockListQuerySize * 1.0022f) + 18000; // 20200 .. 40000
}
else { // More than 10M blocks per query. Most worlds are smaller than this...
querySize = static_cast<int>(m_blockListQuerySize * 1.001f) + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.)
}
int rows = 1; int rows = 1;
for (int offset = 0; rows > 0; offset += m_blockListQuerySize) { for (int offset = 0; rows > 0; offset += m_blockListQuerySize) {
@ -159,9 +175,10 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList()
sqlite3_bind_int(m_blockPosListStatement, 2, offset); sqlite3_bind_int(m_blockPosListStatement, 2, offset);
rows = getBlockPosListRows(); rows = getBlockPosListRows();
sqlite3_reset(m_blockPosListStatement); sqlite3_reset(m_blockPosListStatement);
if (rows > 0) if (rows > 0) {
sleepMs(10); // Be nice to a concurrent user sleepMs(10); // Be nice to a concurrent user
} }
}
m_blockIdSet.clear(); m_blockIdSet.clear();
@ -183,26 +200,31 @@ int DBSQLite3::getBlockPosListRows()
{ {
int rows = 0; int rows = 0;
uint64_t time0 = getRelativeTimeStampMs(); auto time0 = std::chrono::steady_clock::now();
while (true) { while (true) {
int result = sqlite3_step(m_blockPosListStatement); int result = sqlite3_step(m_blockPosListStatement);
if(result == SQLITE_ROW) { if (result == SQLITE_ROW) {
rows++; rows++;
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0); sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1); sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1);
if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) { if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) {
m_blockPosList.push_back(BlockPos(blocknum, rowid)); m_blockPosList.push_back(BlockPos(blocknum, rowid));
} }
} else if (result == SQLITE_BUSY) // Wait some time and try again }
else if (result == SQLITE_BUSY) { // Wait some time and try again
sleepMs(10); sleepMs(10);
else }
else {
break; break;
} }
}
uint64_t time1 = getRelativeTimeStampMs(); auto time1 = std::chrono::steady_clock::now();
if (time1 - time0 > m_blockPosListQueryTime) uint64_t diff = std::chrono::duration_cast<std::chrono::nanoseconds>(time1 - time0).count();
m_blockPosListQueryTime = time1 - time0; if (diff > m_blockPosListQueryTime) {
m_blockPosListQueryTime = diff;
}
return rows; return rows;
} }
@ -219,7 +241,7 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
// Disabled RowID querying, as it may cause blocks not to be found when mapping // Disabled RowID querying, as it may cause blocks not to be found when mapping
// while minetest is running (i.e. modifying blocks). // while minetest is running (i.e. modifying blocks).
if (0 && pos.databasePosIdIsValid()) { if (false && pos.databasePosIdIsValid()) {
statement = m_blockOnRowidStatement; statement = m_blockOnRowidStatement;
sqlite3_bind_int64(m_blockOnRowidStatement, 1, pos.databasePosId()); sqlite3_bind_int64(m_blockOnRowidStatement, 1, pos.databasePosId());
} }
@ -230,15 +252,17 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
while (true) { while (true) {
result = sqlite3_step(statement); result = sqlite3_step(statement);
if(result == SQLITE_ROW) { if (result == SQLITE_ROW) {
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1)); const auto *data = static_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
int size = sqlite3_column_bytes(statement, 1); int size = sqlite3_column_bytes(statement, 1);
block.second.assign(&data[0], &data[size]); block.second.assign(&data[0], &data[size]);
m_blocksReadCount++; m_blocksReadCount++;
break; break;
} else if (result == SQLITE_BUSY) { // Wait some time and try again }
else if (result == SQLITE_BUSY) { // Wait some time and try again
sleepMs(10); sleepMs(10);
} else { }
else {
break; break;
} }
} }

15
Minetestmapper/db-sqlite3.h
View File

@ -5,10 +5,9 @@
#include "db.h" #include "db.h"
#include <sqlite3.h> #include <sqlite3.h>
#include <string>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <string>
#include <sstream>
class DBSQLite3 : public DB { class DBSQLite3 : public DB {
@ -31,11 +30,11 @@ private:
int m_blocksQueriedCount; int m_blocksQueriedCount;
int m_blocksReadCount; int m_blocksReadCount;
sqlite3 *m_db; sqlite3 *m_db = nullptr;
sqlite3_stmt *m_dataVersionStatement; sqlite3_stmt *m_dataVersionStatement = nullptr;
sqlite3_stmt *m_blockPosListStatement; sqlite3_stmt *m_blockPosListStatement = nullptr;
sqlite3_stmt *m_blockOnPosStatement; sqlite3_stmt *m_blockOnPosStatement = nullptr;
sqlite3_stmt *m_blockOnRowidStatement; sqlite3_stmt *m_blockOnRowidStatement = nullptr;
std::ostringstream m_getBlockSetStatementBlocks; std::ostringstream m_getBlockSetStatementBlocks;
BlockCache m_blockCache; BlockCache m_blockCache;
BlockPosList m_blockPosList; BlockPosList m_blockPosList;
@ -44,7 +43,7 @@ private:
uint64_t m_blockPosListQueryTime; uint64_t m_blockPosListQueryTime;
int64_t getDataVersion(); int64_t getDataVersion();
void prepareBlockOnPosStatement(void); void prepareBlockOnPosStatement();
int getBlockPosListRows(); int getBlockPosListRows();
Block getBlockOnPosRaw(const BlockPos &pos); Block getBlockOnPosRaw(const BlockPos &pos);
void cacheBlocks(sqlite3_stmt *SQLstatement); void cacheBlocks(sqlite3_stmt *SQLstatement);