Modernize db-sqlite3
This commit is contained in:
Unknown
parent
467e7cea54
commit
685b136563
@ -2,12 +2,12 @@
|
||||
|
||||
#ifdef USE_SQLITE3
|
||||
|
||||
#include <stdexcept>
|
||||
#include <chrono>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <ctime>
|
||||
#include "porting.h"
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
|
||||
#define DATAVERSION_STATEMENT "PRAGMA data_version"
|
||||
#define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks"
|
||||
@ -18,6 +18,8 @@
|
||||
#define BLOCKLIST_QUERY_SIZE_MIN 2000
|
||||
#define BLOCKLIST_QUERY_SIZE_DEFAULT 250000
|
||||
|
||||
#define sleepMs(x) std::this_thread::sleep_for(std::chrono::milliseconds(x))
|
||||
|
||||
// If zero, a full block list is obtained using a single query.
|
||||
// If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used.
|
||||
int DBSQLite3::m_blockListQuerySize = 0;
|
||||
@ -26,66 +28,72 @@ bool DBSQLite3::warnDatabaseLockDelay = true;
|
||||
|
||||
void DBSQLite3::setLimitBlockListQuerySize(int count)
|
||||
{
|
||||
if (m_firstDatabaseInitialized)
|
||||
if (m_firstDatabaseInitialized) {
|
||||
throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open");
|
||||
}
|
||||
if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) {
|
||||
std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl;
|
||||
count = BLOCKLIST_QUERY_SIZE_MIN;
|
||||
}
|
||||
if (count < 0)
|
||||
if (count < 0) {
|
||||
m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT;
|
||||
else
|
||||
}
|
||||
else {
|
||||
m_blockListQuerySize = count;
|
||||
}
|
||||
}
|
||||
|
||||
DBSQLite3::DBSQLite3(const std::string &mapdir) :
|
||||
m_blocksQueriedCount(0),
|
||||
m_blocksReadCount(0),
|
||||
m_blockPosListStatement(NULL),
|
||||
m_blockOnPosStatement(NULL),
|
||||
m_blockOnRowidStatement(NULL)
|
||||
m_blocksReadCount(0)
|
||||
{
|
||||
|
||||
|
||||
m_firstDatabaseInitialized = true;
|
||||
std::string db_name = mapdir + "map.sqlite";
|
||||
if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
|
||||
if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, nullptr) != SQLITE_OK) {
|
||||
throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
|
||||
}
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT)-1, &m_dataVersionStatement, 0)) {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT) - 1, &m_dataVersionStatement, nullptr)) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)");
|
||||
}
|
||||
if (m_blockListQuerySize == 0) {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)");
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT,
|
||||
sizeof(BLOCKPOSLIST_LIMITED_STATEMENT)-1, &m_blockPosListStatement, 0)) {
|
||||
sizeof(BLOCKPOSLIST_LIMITED_STATEMENT) - 1, &m_blockPosListStatement, nullptr)) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))");
|
||||
}
|
||||
}
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS)-1, &m_blockOnPosStatement, 0)) {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS) - 1, &m_blockOnPosStatement, nullptr)) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)");
|
||||
}
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID)-1, &m_blockOnRowidStatement, 0)) {
|
||||
if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_ROWID, sizeof(BLOCK_STATEMENT_ROWID) - 1, &m_blockOnRowidStatement, nullptr)) {
|
||||
throw std::runtime_error("Failed to prepare SQL statement (blockOnRowidStatement)");
|
||||
}
|
||||
}
|
||||
|
||||
DBSQLite3::~DBSQLite3() {
|
||||
if (m_blockPosListStatement) sqlite3_finalize(m_blockPosListStatement);
|
||||
if (m_blockOnPosStatement) sqlite3_finalize(m_blockOnPosStatement);
|
||||
if (m_blockOnRowidStatement) sqlite3_finalize(m_blockOnRowidStatement);
|
||||
if (m_blockPosListStatement) {
|
||||
sqlite3_finalize(m_blockPosListStatement);
|
||||
}
|
||||
if (m_blockOnPosStatement) {
|
||||
sqlite3_finalize(m_blockOnPosStatement);
|
||||
}
|
||||
if (m_blockOnRowidStatement) {
|
||||
sqlite3_finalize(m_blockOnRowidStatement);
|
||||
}
|
||||
sqlite3_close(m_db);
|
||||
}
|
||||
|
||||
int DBSQLite3::getBlocksReadCount(void)
|
||||
int DBSQLite3::getBlocksReadCount()
|
||||
{
|
||||
return m_blocksReadCount;
|
||||
}
|
||||
|
||||
int DBSQLite3::getBlocksQueriedCount(void)
|
||||
int DBSQLite3::getBlocksQueriedCount()
|
||||
{
|
||||
return m_blocksQueriedCount;
|
||||
}
|
||||
@ -95,12 +103,15 @@ int64_t DBSQLite3::getDataVersion()
|
||||
int64_t version = 0;
|
||||
for (;;) {
|
||||
int result = sqlite3_step(m_dataVersionStatement);
|
||||
if(result == SQLITE_ROW)
|
||||
if (result == SQLITE_ROW) {
|
||||
version = sqlite3_column_int64(m_dataVersionStatement, 0);
|
||||
else if (result == SQLITE_BUSY) // Wait some time and try again
|
||||
}
|
||||
else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
sleepMs(10);
|
||||
else
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_reset(m_blockPosListStatement);
|
||||
return version;
|
||||
@ -142,16 +153,21 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList()
|
||||
// The enforced minimum of 1000 is probably too small to avoid skipped blocks
|
||||
// if heavy world generation is going on, but then, a query size of 10000 is
|
||||
// also pretty small.
|
||||
if (m_blockListQuerySize <= 10000)
|
||||
if (m_blockListQuerySize <= 10000) {
|
||||
querySize = m_blockListQuerySize + 1000;
|
||||
else if (m_blockListQuerySize <= 100000)
|
||||
querySize = m_blockListQuerySize * 1.1; // 1000 .. 10000
|
||||
else if (m_blockListQuerySize <= 1000000)
|
||||
querySize = m_blockListQuerySize * 1.011 + 9000; // 10100 .. 20000
|
||||
else if (m_blockListQuerySize <= 10000000)
|
||||
querySize = m_blockListQuerySize * 1.0022 + 18000; // 20200 .. 40000
|
||||
else // More than 10M blocks per query. Most worlds are smaller than this...
|
||||
querySize = m_blockListQuerySize * 1.001 + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.)
|
||||
}
|
||||
else if (m_blockListQuerySize <= 100000) {
|
||||
querySize = static_cast<int>(m_blockListQuerySize * 1.1f); // 1000 .. 10000
|
||||
}
|
||||
else if (m_blockListQuerySize <= 1000000) {
|
||||
querySize = static_cast<int>(m_blockListQuerySize * 1.011f) + 9000; // 10100 .. 20000
|
||||
}
|
||||
else if (m_blockListQuerySize <= 10000000) {
|
||||
querySize = static_cast<int>(m_blockListQuerySize * 1.0022f) + 18000; // 20200 .. 40000
|
||||
}
|
||||
else { // More than 10M blocks per query. Most worlds are smaller than this...
|
||||
querySize = static_cast<int>(m_blockListQuerySize * 1.001f) + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.)
|
||||
}
|
||||
|
||||
int rows = 1;
|
||||
for (int offset = 0; rows > 0; offset += m_blockListQuerySize) {
|
||||
@ -159,8 +175,9 @@ const DB::BlockPosList &DBSQLite3::getBlockPosList()
|
||||
sqlite3_bind_int(m_blockPosListStatement, 2, offset);
|
||||
rows = getBlockPosListRows();
|
||||
sqlite3_reset(m_blockPosListStatement);
|
||||
if (rows > 0)
|
||||
if (rows > 0) {
|
||||
sleepMs(10); // Be nice to a concurrent user
|
||||
}
|
||||
}
|
||||
|
||||
m_blockIdSet.clear();
|
||||
@ -183,26 +200,31 @@ int DBSQLite3::getBlockPosListRows()
|
||||
{
|
||||
int rows = 0;
|
||||
|
||||
uint64_t time0 = getRelativeTimeStampMs();
|
||||
auto time0 = std::chrono::steady_clock::now();
|
||||
|
||||
while (true) {
|
||||
int result = sqlite3_step(m_blockPosListStatement);
|
||||
if(result == SQLITE_ROW) {
|
||||
if (result == SQLITE_ROW) {
|
||||
rows++;
|
||||
sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
|
||||
sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1);
|
||||
if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) {
|
||||
m_blockPosList.push_back(BlockPos(blocknum, rowid));
|
||||
}
|
||||
} else if (result == SQLITE_BUSY) // Wait some time and try again
|
||||
}
|
||||
else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
sleepMs(10);
|
||||
else
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t time1 = getRelativeTimeStampMs();
|
||||
if (time1 - time0 > m_blockPosListQueryTime)
|
||||
m_blockPosListQueryTime = time1 - time0;
|
||||
auto time1 = std::chrono::steady_clock::now();
|
||||
uint64_t diff = std::chrono::duration_cast<std::chrono::nanoseconds>(time1 - time0).count();
|
||||
if (diff > m_blockPosListQueryTime) {
|
||||
m_blockPosListQueryTime = diff;
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
@ -212,14 +234,14 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
|
||||
{
|
||||
Block block(pos, {});
|
||||
int result = 0;
|
||||
|
||||
|
||||
m_blocksQueriedCount++;
|
||||
|
||||
sqlite3_stmt *statement;
|
||||
|
||||
// Disabled RowID querying, as it may cause blocks not to be found when mapping
|
||||
// while minetest is running (i.e. modifying blocks).
|
||||
if (0 && pos.databasePosIdIsValid()) {
|
||||
if (false && pos.databasePosIdIsValid()) {
|
||||
statement = m_blockOnRowidStatement;
|
||||
sqlite3_bind_int64(m_blockOnRowidStatement, 1, pos.databasePosId());
|
||||
}
|
||||
@ -230,15 +252,17 @@ DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
|
||||
|
||||
while (true) {
|
||||
result = sqlite3_step(statement);
|
||||
if(result == SQLITE_ROW) {
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
|
||||
if (result == SQLITE_ROW) {
|
||||
const auto *data = static_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
|
||||
int size = sqlite3_column_bytes(statement, 1);
|
||||
block.second.assign(&data[0], &data[size]);
|
||||
m_blocksReadCount++;
|
||||
break;
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
}
|
||||
else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
sleepMs(10);
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,10 +5,9 @@
|
||||
|
||||
#include "db.h"
|
||||
#include <sqlite3.h>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
class DBSQLite3 : public DB {
|
||||
@ -31,11 +30,11 @@ private:
|
||||
|
||||
int m_blocksQueriedCount;
|
||||
int m_blocksReadCount;
|
||||
sqlite3 *m_db;
|
||||
sqlite3_stmt *m_dataVersionStatement;
|
||||
sqlite3_stmt *m_blockPosListStatement;
|
||||
sqlite3_stmt *m_blockOnPosStatement;
|
||||
sqlite3_stmt *m_blockOnRowidStatement;
|
||||
sqlite3 *m_db = nullptr;
|
||||
sqlite3_stmt *m_dataVersionStatement = nullptr;
|
||||
sqlite3_stmt *m_blockPosListStatement = nullptr;
|
||||
sqlite3_stmt *m_blockOnPosStatement = nullptr;
|
||||
sqlite3_stmt *m_blockOnRowidStatement = nullptr;
|
||||
std::ostringstream m_getBlockSetStatementBlocks;
|
||||
BlockCache m_blockCache;
|
||||
BlockPosList m_blockPosList;
|
||||
@ -44,7 +43,7 @@ private:
|
||||
uint64_t m_blockPosListQueryTime;
|
||||
|
||||
int64_t getDataVersion();
|
||||
void prepareBlockOnPosStatement(void);
|
||||
void prepareBlockOnPosStatement();
|
||||
int getBlockPosListRows();
|
||||
Block getBlockOnPosRaw(const BlockPos &pos);
|
||||
void cacheBlocks(sqlite3_stmt *SQLstatement);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user