Add an option to prevent keeping sqlite3 database locked for too long

While prescanning a large database, minetestmapper may keep the database
locked for too long, causing minetest to complain, and even bail out
sometimes. Minetest will print warnings like:

	SQLite3 database has been locked for <duration>

The new option --sqlite3-limit-prescan-query-size limits the number
of records queried from the database in a single query, thus limiting
the duration of the lock.

If minetest kept the database locked for 1 second or more, *and* if
the database was modified in that time, minetestmapper will issue a warning,
and suggest using --sqlite3-limit-prescan-query-size.

Changelog

@@ -2,6 +2,16 @@
     Enhancements
 	- Improved rpm platform detection and version string
 	- Allow specifying a packaging version
+	- Added option --sqlite3-limit-prescan-query-size to limit the number
+		of records queried simultaneously during a database prescan.
+		Use this if mapping while minetest is running causes minetest
+		to report warnings like:
+			SQLite3 database has been locked for <duration>
+		This may happen when mapping relatively large worlds.
+
+		If minetestmapper keeps the database locked for too long,
+		*and* if the database was modified in that time, minetestmapper
+		will also issue a warning.
     Bugfixes:
 	- Fixed compilation failure when docutils is not installed
 	- Removed all compiler warnings on Windows

db-sqlite3.cpp

@@ -1,12 +1,40 @@
 #include "db-sqlite3.h"
 #include <stdexcept>
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+#include <ctime>
 #include "porting.h"
 #include "types.h"
 
-#define BLOCKPOSLIST_STATEMENT	"SELECT pos, rowid FROM blocks"
+#define DATAVERSION_STATEMENT		"PRAGMA data_version"
-#define BLOCK_STATEMENT_POS	"SELECT pos, data FROM blocks WHERE pos == ?"
+#define BLOCKPOSLIST_STATEMENT		"SELECT pos, rowid FROM blocks"
-#define BLOCK_STATEMENT_ROWID	"SELECT pos, data FROM blocks WHERE rowid == ?"
+#define BLOCKPOSLIST_LIMITED_STATEMENT	"SELECT pos, rowid FROM blocks ORDER BY pos LIMIT ? OFFSET ?"
+#define BLOCK_STATEMENT_POS		"SELECT pos, data FROM blocks WHERE pos == ?"
+#define BLOCK_STATEMENT_ROWID		"SELECT pos, data FROM blocks WHERE rowid == ?"
 
+#define BLOCKLIST_QUERY_SIZE_MIN	2000
+#define BLOCKLIST_QUERY_SIZE_DEFAULT	250000
+
+// If zero, a full block list is obtained using a single query.
+// If negative, the default value (BLOCKLIST_QUERY_SIZE_DEFAULT) will be used.
+int DBSQLite3::m_blockListQuerySize = 0;
+bool DBSQLite3::m_firstDatabaseInitialized = false;
+bool DBSQLite3::warnDatabaseLockDelay = true;
+
+void DBSQLite3::setLimitBlockListQuerySize(int count)
+{
+	if (m_firstDatabaseInitialized)
+		throw std::runtime_error("Cannot set or change SQLite3 prescan query size: database is already open");
+	if (count > 0 && count < BLOCKLIST_QUERY_SIZE_MIN) {
+		std::cerr << "Limit for SQLite3 prescan query size is too small - increased to " << BLOCKLIST_QUERY_SIZE_MIN << std::endl;
+		count = BLOCKLIST_QUERY_SIZE_MIN;
+	}
+	if (count < 0)
+		m_blockListQuerySize = BLOCKLIST_QUERY_SIZE_DEFAULT;
+	else
+		m_blockListQuerySize = count;
+}
 
 DBSQLite3::DBSQLite3(const std::string &mapdir) :
 	m_blocksQueriedCount(0),
@@ -16,12 +44,24 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) :
 	m_blockOnRowidStatement(NULL)
 {
 	
+	m_firstDatabaseInitialized = true;
 	std::string db_name = mapdir + "map.sqlite";
 	if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
 		throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
 	}
-	if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) {
+	if (SQLITE_OK != sqlite3_prepare_v2(m_db, DATAVERSION_STATEMENT, sizeof(DATAVERSION_STATEMENT)-1, &m_dataVersionStatement, 0)) {
-		throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)");
+		throw std::runtime_error("Failed to prepare SQL statement (dataVersionStatement)");
+	}
+	if (m_blockListQuerySize == 0) {
+		if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_STATEMENT, sizeof(BLOCKPOSLIST_STATEMENT)-1, &m_blockPosListStatement, 0)) {
+			throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement)");
+		}
+	}
+	else {
+		if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCKPOSLIST_LIMITED_STATEMENT,
+				sizeof(BLOCKPOSLIST_LIMITED_STATEMENT)-1, &m_blockPosListStatement, 0)) {
+			throw std::runtime_error("Failed to prepare SQL statement (blockPosListStatement (limited))");
+		}
 	}
 	if (SQLITE_OK != sqlite3_prepare_v2(m_db, BLOCK_STATEMENT_POS, sizeof(BLOCK_STATEMENT_POS)-1, &m_blockOnPosStatement, 0)) {
 		throw std::runtime_error("Failed to prepare SQL statement (blockOnPosStatement)");
@@ -48,23 +88,123 @@ int DBSQLite3::getBlocksQueriedCount(void)
 	return m_blocksQueriedCount;
 }
 
-const DB::BlockPosList &DBSQLite3::getBlockPosList() {
+int64_t DBSQLite3::getDataVersion()
-	m_blockPosList.clear();
+{
-	while (true) {
+	int64_t version = 0;
-		int result = sqlite3_step(m_blockPosListStatement);
+	for (;;) {
-		if(result == SQLITE_ROW) {
+		int result = sqlite3_step(m_dataVersionStatement);
-			sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
+		if(result == SQLITE_ROW)
-			sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1);
+			version = sqlite3_column_int64(m_dataVersionStatement, 0);
-			m_blockPosList.push_back(BlockPos(blocknum, rowid));
+		else if (result == SQLITE_BUSY) // Wait some time and try again
-		} else if (result == SQLITE_BUSY) // Wait some time and try again
 			sleepMs(10);
 		else
 			break;
 	}
 	sqlite3_reset(m_blockPosListStatement);
+	return version;
+}
+
+const DB::BlockPosList &DBSQLite3::getBlockPosList()
+{
+	m_blockPosList.clear();
+
+	m_blockPosListQueryTime = 0;
+	int64_t dataVersionStart = getDataVersion();
+
+	if (!m_blockListQuerySize) {
+
+		getBlockPosListRows();
+		sqlite3_reset(m_blockPosListStatement);
+
+		if (m_blockPosListQueryTime >= 1000 && warnDatabaseLockDelay && getDataVersion() != dataVersionStart) {
+			std::ostringstream oss;
+			oss << "WARNING: "
+				<< "Block list query duration was "
+				<< m_blockPosListQueryTime / 1000 << "."
+				<< std::fixed << std::setw(3) << std::setfill('0')
+				<< m_blockPosListQueryTime % 1000 << " seconds"
+				<< " while another process modified the database. Consider using --sqlite3-limit-prescan-query-size";
+			std::cout << oss.str() << std::endl;
+		}
+
+		return m_blockPosList;
+	}
+
+	// As queries overlap, remember which id's have been seen to avoid duplicates
+	m_blockIdSet.clear();
+
+	int querySize = 0;
+
+	// Select some more blocks than requested, to make sure that newly inserted
+	// blocks don't cause other blocks to be skipped.
+	// The enforced minimum of 1000 is probably too small to avoid skipped blocks
+	// if heavy world generation is going on, but then, a query size of 10000 is
+	// also pretty small.
+	if (m_blockListQuerySize <= 10000)
+		querySize = m_blockListQuerySize + 1000;
+	else if (m_blockListQuerySize <= 100000)
+		querySize = m_blockListQuerySize * 1.1;			// 1000 .. 10000
+	else if (m_blockListQuerySize <= 1000000)
+		querySize = m_blockListQuerySize * 1.011 + 9000;	// 10100 .. 20000
+	else if (m_blockListQuerySize <= 10000000)
+		querySize = m_blockListQuerySize * 1.0022 + 18000;	// 20200 .. 40000
+	else	// More than 10M blocks per query. Most worlds are smaller than this...
+		querySize = m_blockListQuerySize * 1.001 + 30000;	// 40000 .. (130K blocks extra for 100M blocks, etc.)
+
+	int rows = 1;
+	for (int offset = 0; rows > 0; offset += m_blockListQuerySize) {
+		sqlite3_bind_int(m_blockPosListStatement, 1, querySize);
+		sqlite3_bind_int(m_blockPosListStatement, 2, offset);
+		rows = getBlockPosListRows();
+		sqlite3_reset(m_blockPosListStatement);
+		if (rows > 0)
+			sleepMs(10);		// Be nice to a concurrent user
+	}
+
+	m_blockIdSet.clear();
+
+	if (m_blockPosListQueryTime >= 1000 && warnDatabaseLockDelay && getDataVersion() != dataVersionStart) {
+		std::ostringstream oss;
+		oss << "WARNING: "
+			<< "Maximum block list query duration was "
+			<< m_blockPosListQueryTime / 1000 << "."
+			<< std::fixed << std::setw(3) << std::setfill('0')
+			<< m_blockPosListQueryTime % 1000 << " seconds"
+			<< " while another process modified the database. Consider decreasing --sqlite3-limit-prescan-query-size";
+		std::cout << oss.str() << std::endl;
+	}
+
 	return m_blockPosList;
 }
 
+int DBSQLite3::getBlockPosListRows()
+{
+	int rows = 0;
+
+	uint64_t time0 = getRelativeTimeStampMs();
+
+	while (true) {
+		int result = sqlite3_step(m_blockPosListStatement);
+		if(result == SQLITE_ROW) {
+			rows++;
+			sqlite3_int64 blocknum = sqlite3_column_int64(m_blockPosListStatement, 0);
+			sqlite3_int64 rowid = sqlite3_column_int64(m_blockPosListStatement, 1);
+			if (!m_blockListQuerySize || m_blockIdSet.insert(blocknum).second) {
+				m_blockPosList.push_back(BlockPos(blocknum, rowid));
+			}
+		} else if (result == SQLITE_BUSY) // Wait some time and try again
+			sleepMs(10);
+		else
+			break;
+	}
+
+	uint64_t time1 = getRelativeTimeStampMs();
+	if (time1 - time0 > m_blockPosListQueryTime)
+		m_blockPosListQueryTime = time1 - time0;
+
+	return rows;
+}
+
 
 DB::Block DBSQLite3::getBlockOnPos(const BlockPos &pos)
 {

db-sqlite3.h

@@ -5,8 +5,10 @@
 #include <sqlite3.h>
 #if __cplusplus >= 201103L
 #include <unordered_map>
+#include <unordered_set>
 #else
 #include <map>
+#include <set>
 #endif
 #include <string>
 #include <sstream>
@@ -16,8 +18,10 @@
 class DBSQLite3 : public DB {
 #if __cplusplus >= 201103L
 	typedef std::unordered_map<int64_t, ustring>  BlockCache;
+	typedef std::unordered_set<int64_t>  BlockIdSet;
 #else
 	typedef std::map<int64_t, ustring>  BlockCache;
+	typedef std::set<int64_t>  BlockIdSet;
 #endif
 public:
 	DBSQLite3(const std::string &mapdir);
@@ -26,18 +30,30 @@ public:
 	virtual const BlockPosList &getBlockPosList();
 	virtual Block getBlockOnPos(const BlockPos &pos);
 	~DBSQLite3();
+
+	static void setLimitBlockListQuerySize(int count = -1);
+	static bool warnDatabaseLockDelay;
 private:
+	static int m_blockListQuerySize;
+	static bool m_firstDatabaseInitialized;
+
 	int m_blocksQueriedCount;
 	int m_blocksReadCount;
 	sqlite3 *m_db;
+	sqlite3_stmt *m_dataVersionStatement;
 	sqlite3_stmt *m_blockPosListStatement;
 	sqlite3_stmt *m_blockOnPosStatement;
 	sqlite3_stmt *m_blockOnRowidStatement;
 	std::ostringstream  m_getBlockSetStatementBlocks;
 	BlockCache  m_blockCache;
 	BlockPosList m_blockPosList;
+	BlockIdSet m_blockIdSet;		// temporary storage. Only used if m_blockListQuerySize > 0
 
+	uint64_t m_blockPosListQueryTime;
+
+	int64_t getDataVersion();
 	void prepareBlockOnPosStatement(void);
+	int getBlockPosListRows();
 	Block getBlockOnPosRaw(const BlockPos &pos);
 	void cacheBlocks(sqlite3_stmt *SQLstatement);
 };

doc/manual.rst

@@ -155,12 +155,17 @@ is also running (and most probably accessing and modifying the database).
 |Backend	|Support for online mapping					|
 +===============+===============================================================+
 |SQLite3	|Works perfectly since 30 dec 2015, or minetest version		|
-|		|0.4.14 (0.5 ?) and later.					|
+|		|0.4.14 and later.						|
 |		|								|
 |		|Minetest versions before 30 dec 2015 (or: version 0.4.13 and	|
 |		|earlier) probably can't handle concurrent mapping, and		|
 |		|may almost certainly crash with error 'database is locked'.	|
 |		|(but different systems may still behave differently...)	|
+|		|								|
+|		|Minetest versions since 30 dec 2015 (or version 0.4.14 and	|
+|		|later), may still be affected by locking delays, and even	|
+|		|rare crashes. Use `--sqlite3-limit-prescan-query-size`_ if	|
+|		|necessary.							|
 +---------------+---------------------------------------------------------------+
 |PostgreSQL	|Works perfectly.						|
 +---------------+---------------------------------------------------------------+
@@ -177,6 +182,9 @@ database while minetestmapper is running). The older versions of minetest will
 only crash if they find the database temporarily locked when writing (due to
 minetestmapper accessing it). Try at your own risk.
 
+Newer versions of may be affected by delays (i.e. lag). If the database is very large,
+and the prescan query keeps it locked for too long a time, minetest may still bail out.
+
 Command-line Options Summary
 ----------------------------
 
@@ -311,7 +319,7 @@ Feedback / information options:
     * ``--version`` :					Print version ID of minetestmapper
     * ``--verbose[=<n>]`` :				Report world and map statistics (size, dimensions, number of blocks)
     * ``--verbose-search-colors[=n]`` :			Report which colors files are used and/or which locations are searched
-    * ``--silence-suggestions all,prefetch`` :		Do not bother doing suggestions
+    * ``--silence-suggestions <types>`` :		Do not bother doing suggestions
     * ``--progress`` :					Show a progress indicator while generating the map
 
 Miscellaneous options
@@ -321,6 +329,7 @@ Miscellaneous options
     * ``--disable-blocklist-prefetch`` :		Do not prefetch a block list - faster when mapping small parts of large worlds.
     * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` :	Specify the format of the database (needed with --disable-blocklist-prefetch and a LevelDB backend).
     * ``--prescan-world=full|auto|disabled`` :		Specify whether to prescan the world (compute a list of all blocks in the world).
+    * ``--sqlite3-limit-prescan-query-size[=<blocks>]`` :	Limit the size of individual block list queries during a world prescan.
 
 
 Detailed Description of Options
@@ -1213,16 +1222,21 @@ Detailed Description of Options
 	.. image:: images/drawscale-both.png
 	.. image:: images/sidescale-interval.png
 
-``--silence-suggestions all,prefetch``
+``--silence-suggestions <types>``
 ......................................
 	Do not print usage suggestions of the specified types.
 
 	If applicable, minetestmapper may suggest using or adjusting certain options
 	if that may be advantageous. This option disables such messages.
 
-	    :all:	Silence all existing (and future) suggestions there may be.
+	    :all:		Silence all existing (and future) suggestions there may be.
-	    :prefetch:	Do not make suggestions a about the use of --disable-blocklist-prefetch,
+	    :prefetch:		Do not make suggestions a about the use of `--disable-blocklist-prefetch`_,
-			and adjustment of --min-y and --max-y when using --disable-blocklist-prefetch.
+				and adjustment of --min-y and --max-y when using --disable-blocklist-prefetch.
+	    :sqlite3-lock:	Do not suggest using `--sqlite3-limit-prescan-query-size`_.
+
+				This warning will be given if the database was kept locked for 1 second or
+				more while fetching the block list `and` if the database was modified during
+				that time.
 
 ``--sqlite-cacheworldrow``
 ..........................
@@ -1233,6 +1247,59 @@ Detailed Description of Options
 	It is still recognised for compatibility with existing scripts,
 	but it has no effect.
 
+``--sqlite3-limit-prescan-query-size[=<blocks>]``
+.................................................
+	Limit the size of block list queries during a world prescan
+	(see `--prescan-world`_).
+
+	Use this if mapping while minetest is running causes minetest to
+	report warnings like:
+
+	    SQLite3 database has been locked for <duration>
+
+	If minetestmapper locks the database for too long, minetest may even
+	bail out eventually (i.e. crash).
+
+	If `--sqlite3-limit-prescan-query-size` is used, instead of doing a single
+	prescan query, minetestmapper will perform multiple queries, each for a
+	limited number of blocks, thus limiting the duration of the database lock.
+
+	To avoid blocks being skipped, which could happen if minetest has inserted new
+	blocks into the database, every query will overlap with the previous query.
+
+	Sample overlap sizes:
+
+	    ============	==========	==========
+	    Blocks		Overlap		Fraction
+	    ============	==========	==========
+	     2000		 1000		 50%
+	     10000		 1000		 10%
+	     100000		 10000		 10%
+	     250000		 11750		 4.7%
+	     1000000		 20000		 2%
+	    ============	==========	==========
+
+	The default value of `blocks` is 250000, the minimum value is 2000. The
+	minimum overlap is 1000.
+
+	E.g. with `blocks` = 100000 the overlap will be 10000, and minetestmapper will
+	perform the following block list prescan queries:
+
+	    =========		==========	==========
+	    Query nr.		From		To
+	    =========		==========	==========
+	     1			 0		 110000
+	     2			 100000		 210000
+	     3			 200000		 310000
+	     4			 300000		 410000
+	    etc. etc.
+	    =========		==========	==========
+
+	When using small values of `blocks` on fast machines, while minetest is
+	busy generating new parts of the world, the overlap may not be sufficient.
+	It is recommended (and much more efficient) to use a value of at least
+	100000.
+
 ``--tilebordercolor <color>``
 .............................
 	Specify the color to use for drawing tile borders.
@@ -2051,7 +2118,8 @@ More information is available:
 .. _--playercolor: `--playercolor <color>`_
 .. _--prescan-world: `--prescan-world=full\|auto\|disabled`_
 .. _--prescan-world=disabled: `--prescan-world=full\|auto\|disabled`_
-.. _--silence-suggestions: `--silence-suggestions all,prefetch`_
+.. _--silence-suggestions: `--silence-suggestions <types>`_
+.. _--sqlite3-limit-prescan-query-size: `--sqlite3-limit-prescan-query-size[=<blocks>]`_
 .. _--scalecolor: `--scalecolor <color>`_
 .. _--scalefactor: `--scalefactor 1:<n>`_
 .. _--height-level-0: `--height-level-0 <level>`_

mapper.cpp

@@ -22,6 +22,7 @@
 #include "TileGenerator.h"
 #include "PixelAttributes.h"
 #include "util.h"
+#include "db-sqlite3.h"
 
 using namespace std;
 
@@ -48,6 +49,7 @@ using namespace std;
 #define OPT_SILENCE_SUGGESTIONS		0x92
 #define OPT_PRESCAN_WORLD		0x93
 #define OPT_DRAWNODES			0x94
+#define OPT_SQLITE_LIMIT_PRESCAN_QUERY	0x95
 
 #define DRAW_ARROW_LENGTH		10
 #define DRAW_ARROW_ANGLE		30
@@ -147,6 +149,9 @@ void usage()
 			"  --disable-blocklist-prefetch[=force]\n"
 			"  --database-format minetest-i64|freeminer-axyz|mixed|query\n"
 			"  --prescan-world=full|auto|disabled\n"
+			#if USE_SQLITE3
+			"  --sqlite3-limit-prescan-query-size[=n]\n"
+			#endif
 			"  --geometry <geometry>\n"
 			"\t(Warning: has a compatibility mode - see README.rst)\n"
 			"  --cornergeometry <geometry>\n"
@@ -161,7 +166,7 @@ void usage()
 			"  --tilecenter <x>,<y>|world|map\n"
 			"  --scalefactor 1:<n>\n"
 			"  --chunksize <size>\n"
-			"  --silence-suggestions all,prefetch\n"
+			"  --silence-suggestions all,prefetch,sqlite3-lock\n"
 			"  --verbose[=n]\n"
 			"  --verbose-search-colors[=n]\n"
 			"  --progress\n"
@@ -741,6 +746,7 @@ int main(int argc, char *argv[])
 		{"database-format", required_argument, 0, OPT_DATABASE_FORMAT},
 		{"prescan-world", required_argument, 0, OPT_PRESCAN_WORLD},
 		{"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW},
+		{"sqlite3-limit-prescan-query-size", optional_argument, 0, OPT_SQLITE_LIMIT_PRESCAN_QUERY},
 		{"tiles", required_argument, 0, 't'},
 		{"tileorigin", required_argument, 0, 'T'},
 		{"tilecenter", required_argument, 0, 'T'},
@@ -856,6 +862,24 @@ int main(int argc, char *argv[])
 						}
 					}
 					break;
+				case OPT_SQLITE_LIMIT_PRESCAN_QUERY:
+					if (!optarg || !*optarg) {
+						#if USE_SQLITE3
+						DBSQLite3::setLimitBlockListQuerySize();
+						#endif
+					}
+					else {
+						if (!isdigit(optarg[0])) {
+							std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': must be a positive number" << std::endl;
+							usage();
+							exit(1);
+						}
+						#if USE_SQLITE3
+						int size = atoi(optarg);
+						DBSQLite3::setLimitBlockListQuerySize(size);
+						#endif
+					}
+					break;
 				case OPT_HEIGHTMAP:
 					generator.setHeightMap(true);
 					heightMap = true;
@@ -1008,10 +1032,18 @@ int main(int argc, char *argv[])
 						std::string flag;
 						iss >> std::skipws >> flag;
 						while (!iss.fail()) {
-							if (flag == "all")
+							if (flag == "all") {
 								generator.setSilenceSuggestion(SUGGESTION_ALL);
-							else if (flag == "prefetch")
+								DBSQLite3::warnDatabaseLockDelay = false;
+							}
+							else if (flag == "prefetch") {
 								generator.setSilenceSuggestion(SUGGESTION_PREFETCH);
+							}
+							else if (flag == "sqlite3-lock") {
+								#if USE_SQLITE3
+								DBSQLite3::warnDatabaseLockDelay = false;
+								#endif
+							}
 							else {
 								std::cerr << "Invalid flag to '" << long_options[option_index].name << "': '" << flag << "'" << std::endl;
 								usage();