diff --git a/Minetestmapper/db-sqlite3.cpp b/Minetestmapper/db-sqlite3.cpp index d4312c4..8c7e7d7 100644 --- a/Minetestmapper/db-sqlite3.cpp +++ b/Minetestmapper/db-sqlite3.cpp @@ -2,12 +2,12 @@ #ifdef USE_SQLITE3 -#include +#include +#include #include #include -#include -#include -#include "porting.h" +#include +#include #define DATAVERSION_STATEMENT "PRAGMA data_version" #define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks" @@ -18,6 +18,8 @@ #define BLOCKLIST_QUERY_SIZE_MIN 2000 #define BLOCKLIST_QUERY_SIZE_DEFAULT 250000 +#define sleepMs(x) std::this_thread::sleep_for(std::chrono::milliseconds(x)) + // If zero, a full block list is obtained using a single query. // If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used. int DBSQLite3::m_blockListQuerySize = 0; @@ -26,66 +28,72 @@ bool DBSQLite3::warnDatabaseLockDelay = true; void DBSQLite3::setLimitBlockListQuerySize(int count) { - if (m_firstDatabaseInitialized) + if (m_firstDatabaseInitialized) { throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open"); + } if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) { std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl; count = BLOCKLIST_QUERY_SIZE_MIN; } - if (count < 0) + if (count < 0) { m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT; - else + } + else { m_blockListQuerySize = count; + } } DBSQLite3::DBSQLite3(const std::string &mapdir) : m_blocksQueriedCount(0), - m_blocksReadCount(0), - m_blockPosListStatement(NULL), - m_blockOnPosStatement(NULL), - m_blockOnRowidStatement(NULL) + m_blocksReadCount(0) { - + m_firstDatabaseInitialized = true; std::string db_name = mapdir + "map.sqlite"; - if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) { + if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, nullptr) != SQLITE_OK) { throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name); } - if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT)-1, &m_dataVersionStatement, 0)) { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT) - 1, &m_dataVersionStatement, nullptr)) { throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)"); } if (m_blockListQuerySize == 0) { - if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) { throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)"); } } else { if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT, - sizeof(BLOCKPOSLIST_LIMITED_STATEMENT)-1, &m_blockPosListStatement, 0)) { + sizeof(BLOCKPOSLIST_LIMITED_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) { throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))"); } } - if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS)-1, &m_blockOnPosStatement, 0)) { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS) - 1, &m_blockOnPosStatement, nullptr)) { throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)"); } - if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID)-1, &m_blockOnRowidStatement, 0)) { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID) - 1, &m_blockOnRowidStatement, nullptr)) { throw std::runtime_error("Failed to prepare SQL statement (blockOnRowidStatement)"); } } DBSQLite3::~DBSQLite3() { - if (m_blockPosListStatement) sqlite3_finalize(m_blockPosListStatement); - if (m_blockOnPosStatement) sqlite3_finalize(m_blockOnPosStatement); - if (m_blockOnRowidStatement) sqlite3_finalize(m_blockOnRowidStatement); + if (m_blockPosListStatement) { + sqlite3_finalize(m_blockPosListStatement); + } + if (m_blockOnPosStatement) { + sqlite3_finalize(m_blockOnPosStatement); + } + if (m_blockOnRowidStatement) { + sqlite3_finalize(m_blockOnRowidStatement); + } sqlite3_close(m_db); } -int DBSQLite3::getBlocksReadCount(void) +int DBSQLite3::getBlocksReadCount() { return m_blocksReadCount; } -int DBSQLite3::getBlocksQueriedCount(void) +int DBSQLite3::getBlocksQueriedCount() { return m_blocksQueriedCount; } @@ -95,12 +103,15 @@ int64_t DBSQLite3::getDataVersion() int64_t version = 0; for (;;) { int result = sqlite3_step(m_dataVersionStatement); - if(result == SQLITE_ROW) + if (result == SQLITE_ROW) { version = sqlite3_column_int64(m_dataVersionStatement, 0); - else if (result == SQLITE_BUSY) // Wait some time and try again + } + else if (result == SQLITE_BUSY) { // Wait some time and try again sleepMs(10); - else + } + else { break; + } } sqlite3_reset(m_blockPosListStatement); return version; @@ -142,16 +153,21 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList() // The enforced minimum of 1000 is probably too small to avoid skipped blocks // if heavy world generation is going on, but then, a query size of 10000 is // also pretty small. - if (m_blockListQuerySize <= 10000) + if (m_blockListQuerySize <= 10000) { querySize = m_blockListQuerySize + 1000; - else if (m_blockListQuerySize <= 100000) - querySize = m_blockListQuerySize * 1.1; // 1000 .. 10000 - else if (m_blockListQuerySize <= 1000000) - querySize = m_blockListQuerySize * 1.011 + 9000; // 10100 .. 20000 - else if (m_blockListQuerySize <= 10000000) - querySize = m_blockListQuerySize * 1.0022 + 18000; // 20200 .. 40000 - else // More than 10M blocks per query. Most worlds are smaller than this... - querySize = m_blockListQuerySize * 1.001 + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.) + } + else if (m_blockListQuerySize <= 100000) { + querySize = static_cast(m_blockListQuerySize * 1.1f); // 1000 .. 10000 + } + else if (m_blockListQuerySize <= 1000000) { + querySize = static_cast(m_blockListQuerySize * 1.011f) + 9000; // 10100 .. 20000 + } + else if (m_blockListQuerySize <= 10000000) { + querySize = static_cast(m_blockListQuerySize * 1.0022f) + 18000; // 20200 .. 40000 + } + else { // More than 10M blocks per query. Most worlds are smaller than this... + querySize = static_cast(m_blockListQuerySize * 1.001f) + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.) + } int rows = 1; for (int offset = 0; rows > 0; offset += m_blockListQuerySize) { @@ -159,8 +175,9 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList() sqlite3_bind_int(m_blockPosListStatement, 2, offset); rows = getBlockPosListRows(); sqlite3_reset(m_blockPosListStatement); - if (rows > 0) + if (rows > 0) { sleepMs(10); // Be nice to a concurrent user + } } m_blockIdSet.clear(); @@ -183,26 +200,31 @@ int DBSQLite3::getBlockPosListRows() { int rows = 0; - uint64_t time0 = getRelativeTimeStampMs(); + auto time0 = std::chrono::steady_clock::now(); while (true) { int result = sqlite3_step(m_blockPosListStatement); - if(result == SQLITE_ROW) { + if (result == SQLITE_ROW) { rows++; sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0); sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1); if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) { m_blockPosList.push_back(BlockPos(blocknum, rowid)); } - } else if (result == SQLITE_BUSY) // Wait some time and try again + } + else if (result == SQLITE_BUSY) { // Wait some time and try again sleepMs(10); - else + } + else { break; + } } - uint64_t time1 = getRelativeTimeStampMs(); - if (time1 - time0 > m_blockPosListQueryTime) - m_blockPosListQueryTime = time1 - time0; + auto time1 = std::chrono::steady_clock::now(); + uint64_t diff = std::chrono::duration_cast(time1 - time0).count(); + if (diff > m_blockPosListQueryTime) { + m_blockPosListQueryTime = diff; + } return rows; } @@ -212,14 +234,14 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos) { Block block(pos, {}); int result = 0; - + m_blocksQueriedCount++; sqlite3_stmt *statement; // Disabled RowID querying, as it may cause blocks not to be found when mapping // while minetest is running (i.e. modifying blocks). - if (0 && pos.databasePosIdIsValid()) { + if (false && pos.databasePosIdIsValid()) { statement = m_blockOnRowidStatement; sqlite3_bind_int64(m_blockOnRowidStatement, 1, pos.databasePosId()); } @@ -230,15 +252,17 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos) while (true) { result = sqlite3_step(statement); - if(result == SQLITE_ROW) { - const unsigned char *data = reinterpret_cast(sqlite3_column_blob(statement, 1)); + if (result == SQLITE_ROW) { + const auto *data = static_cast(sqlite3_column_blob(statement, 1)); int size = sqlite3_column_bytes(statement, 1); block.second.assign(&data[0], &data[size]); m_blocksReadCount++; break; - } else if (result == SQLITE_BUSY) { // Wait some time and try again + } + else if (result == SQLITE_BUSY) { // Wait some time and try again sleepMs(10); - } else { + } + else { break; } } diff --git a/Minetestmapper/db-sqlite3.h b/Minetestmapper/db-sqlite3.h index 5580814..7fe8948 100644 --- a/Minetestmapper/db-sqlite3.h +++ b/Minetestmapper/db-sqlite3.h @@ -5,10 +5,9 @@ #include "db.h" #include +#include #include #include -#include -#include class DBSQLite3 : public DB { @@ -31,11 +30,11 @@ private: int m_blocksQueriedCount; int m_blocksReadCount; - sqlite3 *m_db; - sqlite3_stmt *m_dataVersionStatement; - sqlite3_stmt *m_blockPosListStatement; - sqlite3_stmt *m_blockOnPosStatement; - sqlite3_stmt *m_blockOnRowidStatement; + sqlite3 *m_db = nullptr; + sqlite3_stmt *m_dataVersionStatement = nullptr; + sqlite3_stmt *m_blockPosListStatement = nullptr; + sqlite3_stmt *m_blockOnPosStatement = nullptr; + sqlite3_stmt *m_blockOnRowidStatement = nullptr; std::ostringstream m_getBlockSetStatementBlocks; BlockCache m_blockCache; BlockPosList m_blockPosList; @@ -44,7 +43,7 @@ private: uint64_t m_blockPosListQueryTime; int64_t getDataVersion(); - void prepareBlockOnPosStatement(void); + void prepareBlockOnPosStatement(); int getBlockPosListRows(); Block getBlockOnPosRaw(const BlockPos &pos); void cacheBlocks(sqlite3_stmt *SQLstatement);