diff --git a/Changelog b/Changelog index 3ff5cc9..abc0d9d 100644 --- a/Changelog +++ b/Changelog @@ -2,6 +2,16 @@ Enhancements - Improved rpm platform detection and version string - Allow specifying a packaging version + - Added option --sqlite3-limit-prescan-query-size to limit the number + of records queried simultaneously during a database prescan. + Use this if mapping while minetest is running causes minetest + to report warnings like: + SQLite3 database has been locked for + This may happen when mapping relatively large worlds. + + If minetestmapper keeps the database locked for too long, + *and* if the database was modified in that time, minetestmapper + will also issue a warning. Bugfixes: - Fixed compilation failure when docutils is not installed - Removed all compiler warnings on Windows diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index acb4f73..6871e8c 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -1,12 +1,40 @@ #include "db-sqlite3.h" #include +#include +#include +#include +#include #include "porting.h" #include "types.h" -#define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks" -#define BLOCK_STATEMENT_POS "SELECT pos, data FROM blocks WHERE pos == ?" -#define BLOCK_STATEMENT_ROWID "SELECT pos, data FROM blocks WHERE rowid == ?" +#define DATAVERSION_STATEMENT "PRAGMA data_version" +#define BLOCKPOSLIST_STATEMENT "SELECT pos, rowid FROM blocks" +#define BLOCKPOSLIST_LIMITED_STATEMENT "SELECT pos, rowid FROM blocks ORDER BY pos LIMIT ? OFFSET ?" +#define BLOCK_STATEMENT_POS "SELECT pos, data FROM blocks WHERE pos == ?" +#define BLOCK_STATEMENT_ROWID "SELECT pos, data FROM blocks WHERE rowid == ?" +#define BLOCKLIST_QUERY_SIZE_MIN 2000 +#define BLOCKLIST_QUERY_SIZE_DEFAULT 250000 + +// If zero, a full block list is obtained using a single query. +// If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used. +int DBSQLite3::m_blockListQuerySize = 0; +bool DBSQLite3::m_firstDatabaseInitialized = false; +bool DBSQLite3::warnDatabaseLockDelay = true; + +void DBSQLite3::setLimitBlockListQuerySize(int count) +{ + if (m_firstDatabaseInitialized) + throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open"); + if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) { + std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl; + count = BLOCKLIST_QUERY_SIZE_MIN; + } + if (count < 0) + m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT; + else + m_blockListQuerySize = count; +} DBSQLite3::DBSQLite3(const std::string &mapdir) : m_blocksQueriedCount(0), @@ -16,12 +44,24 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) : m_blockOnRowidStatement(NULL) { + m_firstDatabaseInitialized = true; std::string db_name = mapdir + "map.sqlite"; if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) { throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name); } - if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) { - throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)"); + if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT)-1, &m_dataVersionStatement, 0)) { + throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)"); + } + if (m_blockListQuerySize == 0) { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) { + throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)"); + } + } + else { + if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT, + sizeof(BLOCKPOSLIST_LIMITED_STATEMENT)-1, &m_blockPosListStatement, 0)) { + throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))"); + } } if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS)-1, &m_blockOnPosStatement, 0)) { throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)"); @@ -48,23 +88,123 @@ int DBSQLite3::getBlocksQueriedCount(void) return m_blocksQueriedCount; } -const DB::BlockPosList &DBSQLite3::getBlockPosList() { - m_blockPosList.clear(); - while (true) { - int result = sqlite3_step(m_blockPosListStatement); - if(result == SQLITE_ROW) { - sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0); - sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1); - m_blockPosList.push_back(BlockPos(blocknum, rowid)); - } else if (result == SQLITE_BUSY) // Wait some time and try again +int64_t DBSQLite3::getDataVersion() +{ + int64_t version = 0; + for (;;) { + int result = sqlite3_step(m_dataVersionStatement); + if(result == SQLITE_ROW) + version = sqlite3_column_int64(m_dataVersionStatement, 0); + else if (result == SQLITE_BUSY) // Wait some time and try again sleepMs(10); else break; } sqlite3_reset(m_blockPosListStatement); + return version; +} + +const DB::BlockPosList &DBSQLite3::getBlockPosList() +{ + m_blockPosList.clear(); + + m_blockPosListQueryTime = 0; + int64_t dataVersionStart = getDataVersion(); + + if (!m_blockListQuerySize) { + + getBlockPosListRows(); + sqlite3_reset(m_blockPosListStatement); + + if (m_blockPosListQueryTime >= 1000 && warnDatabaseLockDelay && getDataVersion() != dataVersionStart) { + std::ostringstream oss; + oss << "WARNING: " + << "Block list query duration was " + << m_blockPosListQueryTime / 1000 << "." + << std::fixed << std::setw(3) << std::setfill('0') + << m_blockPosListQueryTime % 1000 << " seconds" + << " while another process modified the database. Consider using --sqlite3-limit-prescan-query-size"; + std::cout << oss.str() << std::endl; + } + + return m_blockPosList; + } + + // As queries overlap, remember which id's have been seen to avoid duplicates + m_blockIdSet.clear(); + + int querySize = 0; + + // Select some more blocks than requested, to make sure that newly inserted + // blocks don't cause other blocks to be skipped. + // The enforced minimum of 1000 is probably too small to avoid skipped blocks + // if heavy world generation is going on, but then, a query size of 10000 is + // also pretty small. + if (m_blockListQuerySize <= 10000) + querySize = m_blockListQuerySize + 1000; + else if (m_blockListQuerySize <= 100000) + querySize = m_blockListQuerySize * 1.1; // 1000 .. 10000 + else if (m_blockListQuerySize <= 1000000) + querySize = m_blockListQuerySize * 1.011 + 9000; // 10100 .. 20000 + else if (m_blockListQuerySize <= 10000000) + querySize = m_blockListQuerySize * 1.0022 + 18000; // 20200 .. 40000 + else // More than 10M blocks per query. Most worlds are smaller than this... + querySize = m_blockListQuerySize * 1.001 + 30000; // 40000 .. (130K blocks extra for 100M blocks, etc.) + + int rows = 1; + for (int offset = 0; rows > 0; offset += m_blockListQuerySize) { + sqlite3_bind_int(m_blockPosListStatement, 1, querySize); + sqlite3_bind_int(m_blockPosListStatement, 2, offset); + rows = getBlockPosListRows(); + sqlite3_reset(m_blockPosListStatement); + if (rows > 0) + sleepMs(10); // Be nice to a concurrent user + } + + m_blockIdSet.clear(); + + if (m_blockPosListQueryTime >= 1000 && warnDatabaseLockDelay && getDataVersion() != dataVersionStart) { + std::ostringstream oss; + oss << "WARNING: " + << "Maximum block list query duration was " + << m_blockPosListQueryTime / 1000 << "." + << std::fixed << std::setw(3) << std::setfill('0') + << m_blockPosListQueryTime % 1000 << " seconds" + << " while another process modified the database. Consider decreasing --sqlite3-limit-prescan-query-size"; + std::cout << oss.str() << std::endl; + } + return m_blockPosList; } +int DBSQLite3::getBlockPosListRows() +{ + int rows = 0; + + uint64_t time0 = getRelativeTimeStampMs(); + + while (true) { + int result = sqlite3_step(m_blockPosListStatement); + if(result == SQLITE_ROW) { + rows++; + sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0); + sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1); + if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) { + m_blockPosList.push_back(BlockPos(blocknum, rowid)); + } + } else if (result == SQLITE_BUSY) // Wait some time and try again + sleepMs(10); + else + break; + } + + uint64_t time1 = getRelativeTimeStampMs(); + if (time1 - time0 > m_blockPosListQueryTime) + m_blockPosListQueryTime = time1 - time0; + + return rows; +} + DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos) { diff --git a/db-sqlite3.h b/db-sqlite3.h index 06bc5c9..e26b990 100644 --- a/db-sqlite3.h +++ b/db-sqlite3.h @@ -5,8 +5,10 @@ #include #if __cplusplus >= 201103L #include +#include #else #include +#include #endif #include #include @@ -16,8 +18,10 @@ class DBSQLite3 : public DB { #if __cplusplus >= 201103L typedef std::unordered_map BlockCache; + typedef std::unordered_set BlockIdSet; #else typedef std::map BlockCache; + typedef std::set BlockIdSet; #endif public: DBSQLite3(const std::string &mapdir); @@ -26,18 +30,30 @@ public: virtual const BlockPosList &getBlockPosList(); virtual Block getBlockOnPos(const BlockPos &pos); ~DBSQLite3(); + + static void setLimitBlockListQuerySize(int count = -1); + static bool warnDatabaseLockDelay; private: + static int m_blockListQuerySize; + static bool m_firstDatabaseInitialized; + int m_blocksQueriedCount; int m_blocksReadCount; sqlite3 *m_db; + sqlite3_stmt *m_dataVersionStatement; sqlite3_stmt *m_blockPosListStatement; sqlite3_stmt *m_blockOnPosStatement; sqlite3_stmt *m_blockOnRowidStatement; std::ostringstream m_getBlockSetStatementBlocks; BlockCache m_blockCache; BlockPosList m_blockPosList; + BlockIdSet m_blockIdSet; // temporary storage. Only used if m_blockListQuerySize > 0 + uint64_t m_blockPosListQueryTime; + + int64_t getDataVersion(); void prepareBlockOnPosStatement(void); + int getBlockPosListRows(); Block getBlockOnPosRaw(const BlockPos &pos); void cacheBlocks(sqlite3_stmt *SQLstatement); }; diff --git a/doc/manual.rst b/doc/manual.rst index 6bcfd01..21067da 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -155,12 +155,17 @@ is also running (and most probably accessing and modifying the database). |Backend |Support for online mapping | +===============+===============================================================+ |SQLite3 |Works perfectly since 30 dec 2015, or minetest version | -| |0.4.14 (0.5 ?) and later. | +| |0.4.14 and later. | | | | | |Minetest versions before 30 dec 2015 (or: version 0.4.13 and | | |earlier) probably can't handle concurrent mapping, and | | |may almost certainly crash with error 'database is locked'. | | |(but different systems may still behave differently...) | +| | | +| |Minetest versions since 30 dec 2015 (or version 0.4.14 and | +| |later), may still be affected by locking delays, and even | +| |rare crashes. Use `--sqlite3-limit-prescan-query-size`_ if | +| |necessary. | +---------------+---------------------------------------------------------------+ |PostgreSQL |Works perfectly. | +---------------+---------------------------------------------------------------+ @@ -177,6 +182,9 @@ database while minetestmapper is running). The older versions of minetest will only crash if they find the database temporarily locked when writing (due to minetestmapper accessing it). Try at your own risk. +Newer versions of may be affected by delays (i.e. lag). If the database is very large, +and the prescan query keeps it locked for too long a time, minetest may still bail out. + Command-line Options Summary ---------------------------- @@ -311,7 +319,7 @@ Feedback / information options: * ``--version`` : Print version ID of minetestmapper * ``--verbose[=]`` : Report world and map statistics (size, dimensions, number of blocks) * ``--verbose-search-colors[=n]`` : Report which colors files are used and/or which locations are searched - * ``--silence-suggestions all,prefetch`` : Do not bother doing suggestions + * ``--silence-suggestions `` : Do not bother doing suggestions * ``--progress`` : Show a progress indicator while generating the map Miscellaneous options @@ -321,6 +329,7 @@ Miscellaneous options * ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds. * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a LevelDB backend). * ``--prescan-world=full|auto|disabled`` : Specify whether to prescan the world (compute a list of all blocks in the world). + * ``--sqlite3-limit-prescan-query-size[=]`` : Limit the size of individual block list queries during a world prescan. Detailed Description of Options @@ -1213,16 +1222,21 @@ Detailed Description of Options .. image:: images/drawscale-both.png .. image:: images/sidescale-interval.png -``--silence-suggestions all,prefetch`` +``--silence-suggestions `` ...................................... Do not print usage suggestions of the specified types. If applicable, minetestmapper may suggest using or adjusting certain options if that may be advantageous. This option disables such messages. - :all: Silence all existing (and future) suggestions there may be. - :prefetch: Do not make suggestions a about the use of --disable-blocklist-prefetch, - and adjustment of --min-y and --max-y when using --disable-blocklist-prefetch. + :all: Silence all existing (and future) suggestions there may be. + :prefetch: Do not make suggestions a about the use of `--disable-blocklist-prefetch`_, + and adjustment of --min-y and --max-y when using --disable-blocklist-prefetch. + :sqlite3-lock: Do not suggest using `--sqlite3-limit-prescan-query-size`_. + + This warning will be given if the database was kept locked for 1 second or + more while fetching the block list `and` if the database was modified during + that time. ``--sqlite-cacheworldrow`` .......................... @@ -1233,6 +1247,59 @@ Detailed Description of Options It is still recognised for compatibility with existing scripts, but it has no effect. +``--sqlite3-limit-prescan-query-size[=]`` +................................................. + Limit the size of block list queries during a world prescan + (see `--prescan-world`_). + + Use this if mapping while minetest is running causes minetest to + report warnings like: + + SQLite3 database has been locked for + + If minetestmapper locks the database for too long, minetest may even + bail out eventually (i.e. crash). + + If `--sqlite3-limit-prescan-query-size` is used, instead of doing a single + prescan query, minetestmapper will perform multiple queries, each for a + limited number of blocks, thus limiting the duration of the database lock. + + To avoid blocks being skipped, which could happen if minetest has inserted new + blocks into the database, every query will overlap with the previous query. + + Sample overlap sizes: + + ============ ========== ========== + Blocks Overlap Fraction + ============ ========== ========== + 2000 1000 50% + 10000 1000 10% + 100000 10000 10% + 250000 11750 4.7% + 1000000 20000 2% + ============ ========== ========== + + The default value of `blocks` is 250000, the minimum value is 2000. The + minimum overlap is 1000. + + E.g. with `blocks` = 100000 the overlap will be 10000, and minetestmapper will + perform the following block list prescan queries: + + ========= ========== ========== + Query nr. From To + ========= ========== ========== + 1 0 110000 + 2 100000 210000 + 3 200000 310000 + 4 300000 410000 + etc. etc. + ========= ========== ========== + + When using small values of `blocks` on fast machines, while minetest is + busy generating new parts of the world, the overlap may not be sufficient. + It is recommended (and much more efficient) to use a value of at least + 100000. + ``--tilebordercolor `` ............................. Specify the color to use for drawing tile borders. @@ -2051,7 +2118,8 @@ More information is available: .. _--playercolor: `--playercolor `_ .. _--prescan-world: `--prescan-world=full\|auto\|disabled`_ .. _--prescan-world=disabled: `--prescan-world=full\|auto\|disabled`_ -.. _--silence-suggestions: `--silence-suggestions all,prefetch`_ +.. _--silence-suggestions: `--silence-suggestions `_ +.. _--sqlite3-limit-prescan-query-size: `--sqlite3-limit-prescan-query-size[=]`_ .. _--scalecolor: `--scalecolor `_ .. _--scalefactor: `--scalefactor 1:`_ .. _--height-level-0: `--height-level-0 `_ diff --git a/mapper.cpp b/mapper.cpp index 301b03e..d468b56 100644 --- a/mapper.cpp +++ b/mapper.cpp @@ -22,6 +22,7 @@ #include "TileGenerator.h" #include "PixelAttributes.h" #include "util.h" +#include "db-sqlite3.h" using namespace std; @@ -48,6 +49,7 @@ using namespace std; #define OPT_SILENCE_SUGGESTIONS 0x92 #define OPT_PRESCAN_WORLD 0x93 #define OPT_DRAWNODES 0x94 +#define OPT_SQLITE_LIMIT_PRESCAN_QUERY 0x95 #define DRAW_ARROW_LENGTH 10 #define DRAW_ARROW_ANGLE 30 @@ -147,6 +149,9 @@ void usage() " --disable-blocklist-prefetch[=force]\n" " --database-format minetest-i64|freeminer-axyz|mixed|query\n" " --prescan-world=full|auto|disabled\n" + #if USE_SQLITE3 + " --sqlite3-limit-prescan-query-size[=n]\n" + #endif " --geometry \n" "\t(Warning: has a compatibility mode - see README.rst)\n" " --cornergeometry \n" @@ -161,7 +166,7 @@ void usage() " --tilecenter ,|world|map\n" " --scalefactor 1:\n" " --chunksize \n" - " --silence-suggestions all,prefetch\n" + " --silence-suggestions all,prefetch,sqlite3-lock\n" " --verbose[=n]\n" " --verbose-search-colors[=n]\n" " --progress\n" @@ -741,6 +746,7 @@ int main(int argc, char *argv[]) {"database-format", required_argument, 0, OPT_DATABASE_FORMAT}, {"prescan-world", required_argument, 0, OPT_PRESCAN_WORLD}, {"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW}, + {"sqlite3-limit-prescan-query-size", optional_argument, 0, OPT_SQLITE_LIMIT_PRESCAN_QUERY}, {"tiles", required_argument, 0, 't'}, {"tileorigin", required_argument, 0, 'T'}, {"tilecenter", required_argument, 0, 'T'}, @@ -856,6 +862,24 @@ int main(int argc, char *argv[]) } } break; + case OPT_SQLITE_LIMIT_PRESCAN_QUERY: + if (!optarg || !*optarg) { + #if USE_SQLITE3 + DBSQLite3::setLimitBlockListQuerySize(); + #endif + } + else { + if (!isdigit(optarg[0])) { + std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': must be a positive number" << std::endl; + usage(); + exit(1); + } + #if USE_SQLITE3 + int size = atoi(optarg); + DBSQLite3::setLimitBlockListQuerySize(size); + #endif + } + break; case OPT_HEIGHTMAP: generator.setHeightMap(true); heightMap = true; @@ -1008,10 +1032,18 @@ int main(int argc, char *argv[]) std::string flag; iss >> std::skipws >> flag; while (!iss.fail()) { - if (flag == "all") + if (flag == "all") { generator.setSilenceSuggestion(SUGGESTION_ALL); - else if (flag == "prefetch") + DBSQLite3::warnDatabaseLockDelay = false; + } + else if (flag == "prefetch") { generator.setSilenceSuggestion(SUGGESTION_PREFETCH); + } + else if (flag == "sqlite3-lock") { + #if USE_SQLITE3 + DBSQLite3::warnDatabaseLockDelay = false; + #endif + } else { std::cerr << "Invalid flag to '" << long_options[option_index].name << "': '" << flag << "'" << std::endl; usage();