Add an option to disable prefetching the block list from the database

This introduces a speed tradeoff.

When mapping a small part of a large world, the prefetch time dominates
the mapping time, and it is more advantageous to skip the prefetch and
query all possible blocks in the mapped space.

When mapping a large fraction of a world, in particular when a lot of
the mapped space is empty, the time spent querying non-existing blocks can
dominate mapping time, and it is more advantageous to prefetch the list
of existing blocks, so that querying huge numbers of non-existing blocks
can be avoided.

Changelog

@@ -1,3 +1,15 @@
+[]
+    Enhancements:
+	- Added an optimisation option: --disable-blocklist-prefetch. With this
+		option, minetestmapper does not query the database to make a block
+		list before starting map generation, but directly tries to fetch
+		all possible blocks that might be in the mapped space.
+
+		This improves speed on large worlds when mapping a small part of
+		them. When mapping a large part of the world, or when not adequately
+		limiting the mapping height (--min-y and --max-y), this option can
+		cause **excessively** long mapping times.
+
 [2 mar 2015]
     Features:
 

TileGenerator.cpp

@@ -34,6 +34,10 @@
 #endif
 
 #define MESSAGE_WIDTH 25
+#define MAX_NOPREFETCH_VOLUME (1LL<<24)
+#define MIN_NOPREFETCH_VOLUME (1LL<<16)
+#define MAX_NOPREFETCH_VOLUME_EXAMPLE "16384x256x16384"
+
 
 using namespace std;
 
@@ -157,6 +161,7 @@ TileGenerator::TileGenerator():
 	m_sideScaleMinor(0),
 	m_heightScaleMajor(0),
 	m_heightScaleMinor(0),
+	m_generateNoPrefetch(0),
 	m_image(0),
 	m_xMin(INT_MAX/16-1),
 	m_xMax(INT_MIN/16+1),
@@ -200,6 +205,11 @@ TileGenerator::~TileGenerator()
 {
 }
 
+void TileGenerator::setGenerateNoPrefetch(int enable)
+{
+	m_generateNoPrefetch = enable;
+}
+
 void TileGenerator::setHeightMap(bool enable)
 {
 	m_heightMap = enable;
@@ -913,6 +923,34 @@ void TileGenerator::loadBlocks()
 	geomYMax = MAPBLOCK_MIN;
 	m_worldBlocks = 0;
 	map_blocks = 0;
+	if (m_generateNoPrefetch) {
+		if (m_generateNoPrefetch == 1) {
+			long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1);
+			if (volume > MAX_NOPREFETCH_VOLUME) {
+				std::ostringstream oss;
+				oss << "Requested map volume is excessive for --disable-blocklist-prefetch: " << volume
+					    << " (" << (m_reqXMax - m_reqXMin + 1)
+					    << " x " << (m_reqYMax - m_reqYMin + 1)
+					    << " x " << (m_reqZMax - m_reqZMin + 1)
+					    << " blocks of 16x16x16 nodes);"
+					    << "\n"
+					    << " Mapping will be slow. Use --disable-blocklist-prefetch='force' for more than " << MAX_NOPREFETCH_VOLUME << " blocks"
+					    << " (e.g. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)";
+				throw(std::runtime_error(oss.str()));
+			}
+		}
+		if (m_shrinkGeometry) {
+			std::cerr << "WARNING: geometrymode 'shrink' not supported with '--disable-blocklist-prefetch'" << std::endl;
+			m_shrinkGeometry = false;
+		}
+		m_xMin = m_reqXMin;
+		m_xMax = m_reqXMax;
+		m_yMin = m_reqYMin;
+		m_yMax = m_reqYMax;
+		m_zMin = m_reqZMin;
+		m_zMax = m_reqZMax;
+	}
+	else {
 	if (progressIndicator)
 		cout << "Scanning world (reading block list)...\r" << std::flush;
 	const DB::BlockPosList &blocks = m_db->getBlockPos();
@@ -1047,6 +1085,8 @@ void TileGenerator::loadBlocks()
 			<< std::setw(6) << "z"
 			<< ")\n";
 	}
+	m_positions.sort();
+	}
 	if (verboseCoordinates >= 1) {
 		cout
 			<< std::setw(MESSAGE_WIDTH) << std::left
@@ -1069,7 +1109,6 @@ void TileGenerator::loadBlocks()
 			<< ")    blocks: "
 			<< std::setw(10) << map_blocks << "\n";
 	}
-	m_positions.sort();
 	if (m_backend == "leveldb") {
 		if (verboseStatistics >= 3) {
 			cout
@@ -1504,6 +1543,51 @@ void TileGenerator::processMapBlock(const DB::Block &block)
 	renderMapBlock(mapData, pos, version);
 }
 
+class MapBlockIterator
+{
+public:
+	virtual ~MapBlockIterator(void) {}
+	virtual MapBlockIterator &operator++(void) = 0;
+	virtual BlockPos &operator*(void) = 0;
+	virtual MapBlockIterator &operator=(const MapBlockIterator &i) = 0;
+	virtual bool operator==(const MapBlockIterator &i) const = 0;
+	bool operator!=(const MapBlockIterator &i) const { return !operator==(i); }
+	virtual void breakDim(int i) { (void) i; }
+};
+
+class MapBlockIteratorBlockList : public MapBlockIterator
+{
+public:
+	MapBlockIteratorBlockList(void) {}
+	MapBlockIteratorBlockList(const std::list<BlockPos>::iterator &i) : m_iter(i) {}
+	MapBlockIterator &operator++(void) override { m_iter++; return *this; }
+	BlockPos &operator*(void) override { return *m_iter; }
+	MapBlockIterator &operator=(const MapBlockIterator &i) override
+		{ const MapBlockIteratorBlockList &i2 = dynamic_cast<const MapBlockIteratorBlockList &>(i); m_iter = i2.m_iter; return *this; }
+	bool operator==(const MapBlockIterator &i) const override
+		{ const MapBlockIteratorBlockList &i2 = dynamic_cast<const MapBlockIteratorBlockList &>(i); return m_iter == i2.m_iter; }
+	// breakDim() might be implemented, but is not strictly necessary
+private:
+	std::list<BlockPos>::iterator m_iter;
+};
+
+class MapBlockIteratorBlockPos : public MapBlockIterator
+{
+public:
+	MapBlockIteratorBlockPos(void) {}
+	MapBlockIteratorBlockPos(const BlockPosIterator &i) : m_iter(i) {}
+	MapBlockIterator &operator++(void) override { m_iter++; return *this; }
+	BlockPos &operator*(void) override { return *m_iter; }
+	MapBlockIterator &operator=(const MapBlockIterator &i) override
+		{ const MapBlockIteratorBlockPos &i2 = dynamic_cast<const MapBlockIteratorBlockPos &>(i); m_iter = i2.m_iter; return *this; }
+	bool operator==(const MapBlockIterator &i) const override
+		{ const MapBlockIteratorBlockPos &i2 = dynamic_cast<const MapBlockIteratorBlockPos &>(i); return m_iter == i2.m_iter; }
+	void breakDim(int i) { m_iter.breakDim(i); }
+private:
+	BlockPosIterator m_iter;
+};
+
+
 void TileGenerator::renderMap()
 {
 	int unpackErrors = 0;
@@ -1514,8 +1598,28 @@ void TileGenerator::renderMap()
 	currentPos.y() = INT_MAX;
 	currentPos.z() = INT_MIN;
 	bool allReaded = false;
-	for (std::list<BlockPos>::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
+	MapBlockIterator *position;
-		const BlockPos &pos = *position;
+	MapBlockIterator *begin;
+	MapBlockIterator *end;
+	if (m_generateNoPrefetch) {
+		position = new MapBlockIteratorBlockPos();
+		begin = new MapBlockIteratorBlockPos(BlockPosIterator(
+				BlockPos(m_xMin, m_yMax, m_zMax),
+				BlockPos(m_xMax, m_yMin, m_zMin)));
+		end = new MapBlockIteratorBlockPos(BlockPosIterator(
+				BlockPos(m_xMin, m_yMax, m_zMax),
+				BlockPos(m_xMax, m_yMin, m_zMin),
+				BlockPosIterator::End));
+	}
+	else {
+		position = new MapBlockIteratorBlockList(std::list<BlockPos>::iterator());
+		begin = new MapBlockIteratorBlockList(m_positions.begin());
+		end = new MapBlockIteratorBlockList(m_positions.end());
+	}
+	std::cout << std::flush;
+	std::cerr << std::flush;
+	for (*position = *begin; *position != *end; ++*position) {
+		const BlockPos &pos = **position;
 		if (currentPos.x() != pos.x() || currentPos.z() != pos.z()) {
 			area_rendered++;
 			if (currentPos.y() == m_yMin)
@@ -1542,6 +1646,7 @@ void TileGenerator::renderMap()
 			currentPos = pos;
 		}
 		else if (allReaded) {
+			position->breakDim(1);
 			continue;
 		}
 		currentPos.y() = pos.y();
@@ -1582,6 +1687,9 @@ void TileGenerator::renderMap()
 			throw(std::runtime_error("Too many block unpacking errors - bailing out"));
 		}
 	}
+	delete position;
+	delete begin;
+	delete end;
 	if (currentPos.z() != INT_MIN) {
 		if (currentPos.y() == m_yMin)
 			m_emptyMapArea++;

TileGenerator.h

@@ -109,6 +109,7 @@ public:
 
 	TileGenerator();
 	~TileGenerator();
+	void setGenerateNoPrefetch(int enable);
 	void setHeightMap(bool enable);
 	void setHeightMapYScale(float scale);
 	void setSeaLevel(int level);
@@ -241,6 +242,7 @@ private:
 	int m_heightScaleMinor;
 
 	DB *m_db;
+	bool m_generateNoPrefetch;
 	long long m_databaseFormatFound[BlockPos::STRFORMAT_MAX];
 	gdImagePtr m_image;
 	PixelAttributes m_blockPixelAttributes;

doc/features.rst

@@ -35,6 +35,8 @@ Minor Features
 * Draw shades to accentuate height differences (on by default)
 * Report actual world dimensions in all directions, as
   well which part of it will be in the map.
+* optionally, avoid reading the block list from the database
+  (may be more efficient when mapping small parts of the *existing* world)
 
 Differences From Stock Minetestmapper
 =====================================
@@ -66,6 +68,8 @@ Differences From Stock Minetestmapper
 * The scale can be enabled on the left and top side individually
 * Major and minor (tick) intervals are configurable for the scale
 * Block numbers are shown on the scale as well
+* optionally, avoid reading the block list from the database
+  (dramatically speeds up generating maps of small parts of large worlds)
 
 In addition a number bugs have been fixed. As bugs are also getting
 fixed in the stock version of minetestmapper, no accurate list

doc/manual.rst

@@ -288,6 +288,7 @@ Miscellaneous options
 .....................
 
     * ``--backend auto|sqlite3|leveldb|redis`` :	Specify or override the database backend to use
+    * ``--disable-blocklist-prefetch`` :		Do not prefetch a block list - faster when mapping small parts of large worlds.
 
 
 Detailed Description of Options
@@ -376,6 +377,40 @@ Detailed Description of Options
 
 	See also `--geometry`_
 
+``--disable-blocklist-prefetch``
+......................................
+	Do not prefetch a list of block coordinates from the database before commencing
+	map generation.
+
+	This option will probably improve mapping speed when mapping a smaller part
+	of a very large world. In other cases it may actually reduce mapping speed.
+	It is incompatible with, and  disables, the 'shrinking' mode of `--geometrymode`_.
+	It also significantly reduces the amount of information the `--verbose`_ option
+	can report.
+
+	Normally, minetestmapper will read a full list of coordinates (not the contents)
+	of existing blocks from the database before starting map generation. This option
+	disables this query, and instead, causes and all blocks that are in the mapped
+	space to be requested individually, whether or not they are in the database.
+
+	Querying the database for a block coordinate list beforehand is time-consuming
+	on large databases. If just a small part of a large world is being mapped, the
+	time for this step quickly dominates the map generation time.
+
+	On the other hand, querying the database for large numbers of non-existing blocks
+	while mapping (possibly several orders of magniture more than there are existing
+	blocks!) is also quite inefficient. If a large part of the blocks queried are not
+	in the database, the cost of those extra queries will quickly dominate map generation
+	time.
+
+	The tradeoff between those two approaches depends on the volume being mapped, the
+	speed of the disk (or SSD), the database backend being used, the number of blocks
+	in the database, etc.
+
+	The worst-case behavior of this option is probably quite bad, even though it will
+	refuse to continue if the requested space is excessive: exceeding 1G (2^30) blocks.
+	Please use this option with consideration, and use `--progress`_ to monitor its
+	actual behavior.
 
 ``--draw[map]<figure> "<geometry> <color> [<text>]"``
 .....................................................

mapper.cpp

@@ -39,6 +39,7 @@ using namespace std;
 #define OPT_DRAWHEIGHTSCALE		0x8d
 #define OPT_SCALEFACTOR			0x8e
 #define OPT_SCALEINTERVAL		0x8f
+#define OPT_NO_BLOCKLIST_PREFETCH	0x90
 
 // Will be replaced with the actual name and location of the executable (if found)
 string executableName = "minetestmapper";
@@ -106,6 +107,7 @@ void usage()
 			"  --min-y <y>\n"
 			"  --max-y <y>\n"
 			"  --backend <" USAGE_DATABASES ">\n"
+			"  --disable-blocklist-prefetch[=force]\n"
 			"  --geometry <geometry>\n"
 			"\t(Warning: has a compatibility mode - see README.rst)\n"
 			"  --cornergeometry <geometry>\n"
@@ -618,6 +620,7 @@ int main(int argc, char *argv[])
 		{"min-y", required_argument, 0, 'a'},
 		{"max-y", required_argument, 0, 'c'},
 		{"backend", required_argument, 0, 'd'},
+		{"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH},
 		{"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW},
 		{"tiles", required_argument, 0, 't'},
 		{"tileorigin", required_argument, 0, 'T'},
@@ -682,6 +685,20 @@ int main(int argc, char *argv[])
 				case 'b':
 					generator.setBgColor(Color(optarg, 0));
 					break;
+				case OPT_NO_BLOCKLIST_PREFETCH:
+					if (optarg && *optarg) {
+						if (std::string(optarg) == "force")
+							generator.setGenerateNoPrefetch(2);
+						else {
+							std::cerr << "Invalid parameter to '" << long_options[option_index].name << "'; expected 'force' or nothing." << std::endl;
+							usage();
+							exit(1);
+						}
+					}
+					else {
+						generator.setGenerateNoPrefetch(1);
+					}
+					break;
 				case OPT_HEIGHTMAP:
 					generator.setHeightMap(true);
 					heightMap = true;