From 38f4272dd3906fb1ed4e706c353390d54a22ca20 Mon Sep 17 00:00:00 2001 From: Rogier Date: Tue, 24 Feb 2015 22:12:41 +0100 Subject: [PATCH] Add an option to disable prefetching the block list from the database This introduces a speed tradeoff. When mapping a small part of a large world, the prefetch time dominates the mapping time, and it is more advantageous to skip the prefetch and query all possible blocks in the mapped space. When mapping a large fraction of a world, in particular when a lot of the mapped space is empty, the time spent querying non-existing blocks can dominate mapping time, and it is more advantageous to prefetch the list of existing blocks, so that querying huge numbers of non-existing blocks can be avoided. --- Changelog | 12 +++++ TileGenerator.cpp | 114 ++++++++++++++++++++++++++++++++++++++++++++-- TileGenerator.h | 2 + doc/features.rst | 4 ++ doc/manual.rst | 35 ++++++++++++++ mapper.cpp | 17 +++++++ 6 files changed, 181 insertions(+), 3 deletions(-) diff --git a/Changelog b/Changelog index a09014d..853fbc7 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,15 @@ +[] + Enhancements: + - Added an optimisation option: --disable-blocklist-prefetch. With this + option, minetestmapper does not query the database to make a block + list before starting map generation, but directly tries to fetch + all possible blocks that might be in the mapped space. + + This improves speed on large worlds when mapping a small part of + them. When mapping a large part of the world, or when not adequately + limiting the mapping height (--min-y and --max-y), this option can + cause **excessively** long mapping times. + [2 mar 2015] Features: diff --git a/TileGenerator.cpp b/TileGenerator.cpp index ce2f186..7e61c27 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -34,6 +34,10 @@ #endif #define MESSAGE_WIDTH 25 +#define MAX_NOPREFETCH_VOLUME (1LL<<24) +#define MIN_NOPREFETCH_VOLUME (1LL<<16) +#define MAX_NOPREFETCH_VOLUME_EXAMPLE "16384x256x16384" + using namespace std; @@ -157,6 +161,7 @@ TileGenerator::TileGenerator(): m_sideScaleMinor(0), m_heightScaleMajor(0), m_heightScaleMinor(0), + m_generateNoPrefetch(0), m_image(0), m_xMin(INT_MAX/16-1), m_xMax(INT_MIN/16+1), @@ -200,6 +205,11 @@ TileGenerator::~TileGenerator() { } +void TileGenerator::setGenerateNoPrefetch(int enable) +{ + m_generateNoPrefetch = enable; +} + void TileGenerator::setHeightMap(bool enable) { m_heightMap = enable; @@ -913,6 +923,34 @@ void TileGenerator::loadBlocks() geomYMax = MAPBLOCK_MIN; m_worldBlocks = 0; map_blocks = 0; + if (m_generateNoPrefetch) { + if (m_generateNoPrefetch == 1) { + long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1); + if (volume > MAX_NOPREFETCH_VOLUME) { + std::ostringstream oss; + oss << "Requested map volume is excessive for --disable-blocklist-prefetch: " << volume + << " (" << (m_reqXMax - m_reqXMin + 1) + << " x " << (m_reqYMax - m_reqYMin + 1) + << " x " << (m_reqZMax - m_reqZMin + 1) + << " blocks of 16x16x16 nodes);" + << "\n" + << " Mapping will be slow. Use --disable-blocklist-prefetch='force' for more than " << MAX_NOPREFETCH_VOLUME << " blocks" + << " (e.g. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)"; + throw(std::runtime_error(oss.str())); + } + } + if (m_shrinkGeometry) { + std::cerr << "WARNING: geometrymode 'shrink' not supported with '--disable-blocklist-prefetch'" << std::endl; + m_shrinkGeometry = false; + } + m_xMin = m_reqXMin; + m_xMax = m_reqXMax; + m_yMin = m_reqYMin; + m_yMax = m_reqYMax; + m_zMin = m_reqZMin; + m_zMax = m_reqZMax; + } + else { if (progressIndicator) cout << "Scanning world (reading block list)...\r" << std::flush; const DB::BlockPosList &blocks = m_db->getBlockPos(); @@ -1047,6 +1085,8 @@ void TileGenerator::loadBlocks() << std::setw(6) << "z" << ")\n"; } + m_positions.sort(); + } if (verboseCoordinates >= 1) { cout << std::setw(MESSAGE_WIDTH) << std::left @@ -1069,7 +1109,6 @@ void TileGenerator::loadBlocks() << ") blocks: " << std::setw(10) << map_blocks << "\n"; } - m_positions.sort(); if (m_backend == "leveldb") { if (verboseStatistics >= 3) { cout @@ -1504,6 +1543,51 @@ void TileGenerator::processMapBlock(const DB::Block &block) renderMapBlock(mapData, pos, version); } +class MapBlockIterator +{ +public: + virtual ~MapBlockIterator(void) {} + virtual MapBlockIterator &operator++(void) = 0; + virtual BlockPos &operator*(void) = 0; + virtual MapBlockIterator &operator=(const MapBlockIterator &i) = 0; + virtual bool operator==(const MapBlockIterator &i) const = 0; + bool operator!=(const MapBlockIterator &i) const { return !operator==(i); } + virtual void breakDim(int i) { (void) i; } +}; + +class MapBlockIteratorBlockList : public MapBlockIterator +{ +public: + MapBlockIteratorBlockList(void) {} + MapBlockIteratorBlockList(const std::list::iterator &i) : m_iter(i) {} + MapBlockIterator &operator++(void) override { m_iter++; return *this; } + BlockPos &operator*(void) override { return *m_iter; } + MapBlockIterator &operator=(const MapBlockIterator &i) override + { const MapBlockIteratorBlockList &i2 = dynamic_cast(i); m_iter = i2.m_iter; return *this; } + bool operator==(const MapBlockIterator &i) const override + { const MapBlockIteratorBlockList &i2 = dynamic_cast(i); return m_iter == i2.m_iter; } + // breakDim() might be implemented, but is not strictly necessary +private: + std::list::iterator m_iter; +}; + +class MapBlockIteratorBlockPos : public MapBlockIterator +{ +public: + MapBlockIteratorBlockPos(void) {} + MapBlockIteratorBlockPos(const BlockPosIterator &i) : m_iter(i) {} + MapBlockIterator &operator++(void) override { m_iter++; return *this; } + BlockPos &operator*(void) override { return *m_iter; } + MapBlockIterator &operator=(const MapBlockIterator &i) override + { const MapBlockIteratorBlockPos &i2 = dynamic_cast(i); m_iter = i2.m_iter; return *this; } + bool operator==(const MapBlockIterator &i) const override + { const MapBlockIteratorBlockPos &i2 = dynamic_cast(i); return m_iter == i2.m_iter; } + void breakDim(int i) { m_iter.breakDim(i); } +private: + BlockPosIterator m_iter; +}; + + void TileGenerator::renderMap() { int unpackErrors = 0; @@ -1514,8 +1598,28 @@ void TileGenerator::renderMap() currentPos.y() = INT_MAX; currentPos.z() = INT_MIN; bool allReaded = false; - for (std::list::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) { - const BlockPos &pos = *position; + MapBlockIterator *position; + MapBlockIterator *begin; + MapBlockIterator *end; + if (m_generateNoPrefetch) { + position = new MapBlockIteratorBlockPos(); + begin = new MapBlockIteratorBlockPos(BlockPosIterator( + BlockPos(m_xMin, m_yMax, m_zMax), + BlockPos(m_xMax, m_yMin, m_zMin))); + end = new MapBlockIteratorBlockPos(BlockPosIterator( + BlockPos(m_xMin, m_yMax, m_zMax), + BlockPos(m_xMax, m_yMin, m_zMin), + BlockPosIterator::End)); + } + else { + position = new MapBlockIteratorBlockList(std::list::iterator()); + begin = new MapBlockIteratorBlockList(m_positions.begin()); + end = new MapBlockIteratorBlockList(m_positions.end()); + } + std::cout << std::flush; + std::cerr << std::flush; + for (*position = *begin; *position != *end; ++*position) { + const BlockPos &pos = **position; if (currentPos.x() != pos.x() || currentPos.z() != pos.z()) { area_rendered++; if (currentPos.y() == m_yMin) @@ -1542,6 +1646,7 @@ void TileGenerator::renderMap() currentPos = pos; } else if (allReaded) { + position->breakDim(1); continue; } currentPos.y() = pos.y(); @@ -1582,6 +1687,9 @@ void TileGenerator::renderMap() throw(std::runtime_error("Too many block unpacking errors - bailing out")); } } + delete position; + delete begin; + delete end; if (currentPos.z() != INT_MIN) { if (currentPos.y() == m_yMin) m_emptyMapArea++; diff --git a/TileGenerator.h b/TileGenerator.h index 7d2a2de..f353ad7 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -109,6 +109,7 @@ public: TileGenerator(); ~TileGenerator(); + void setGenerateNoPrefetch(int enable); void setHeightMap(bool enable); void setHeightMapYScale(float scale); void setSeaLevel(int level); @@ -241,6 +242,7 @@ private: int m_heightScaleMinor; DB *m_db; + bool m_generateNoPrefetch; long long m_databaseFormatFound[BlockPos::STRFORMAT_MAX]; gdImagePtr m_image; PixelAttributes m_blockPixelAttributes; diff --git a/doc/features.rst b/doc/features.rst index d39f221..182975f 100644 --- a/doc/features.rst +++ b/doc/features.rst @@ -35,6 +35,8 @@ Minor Features * Draw shades to accentuate height differences (on by default) * Report actual world dimensions in all directions, as well which part of it will be in the map. +* optionally, avoid reading the block list from the database + (may be more efficient when mapping small parts of the *existing* world) Differences From Stock Minetestmapper ===================================== @@ -66,6 +68,8 @@ Differences From Stock Minetestmapper * The scale can be enabled on the left and top side individually * Major and minor (tick) intervals are configurable for the scale * Block numbers are shown on the scale as well +* optionally, avoid reading the block list from the database + (dramatically speeds up generating maps of small parts of large worlds) In addition a number bugs have been fixed. As bugs are also getting fixed in the stock version of minetestmapper, no accurate list diff --git a/doc/manual.rst b/doc/manual.rst index 53054c2..6dfc7f3 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -288,6 +288,7 @@ Miscellaneous options ..................... * ``--backend auto|sqlite3|leveldb|redis`` : Specify or override the database backend to use + * ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds. Detailed Description of Options @@ -376,6 +377,40 @@ Detailed Description of Options See also `--geometry`_ +``--disable-blocklist-prefetch`` +...................................... + Do not prefetch a list of block coordinates from the database before commencing + map generation. + + This option will probably improve mapping speed when mapping a smaller part + of a very large world. In other cases it may actually reduce mapping speed. + It is incompatible with, and disables, the 'shrinking' mode of `--geometrymode`_. + It also significantly reduces the amount of information the `--verbose`_ option + can report. + + Normally, minetestmapper will read a full list of coordinates (not the contents) + of existing blocks from the database before starting map generation. This option + disables this query, and instead, causes and all blocks that are in the mapped + space to be requested individually, whether or not they are in the database. + + Querying the database for a block coordinate list beforehand is time-consuming + on large databases. If just a small part of a large world is being mapped, the + time for this step quickly dominates the map generation time. + + On the other hand, querying the database for large numbers of non-existing blocks + while mapping (possibly several orders of magniture more than there are existing + blocks!) is also quite inefficient. If a large part of the blocks queried are not + in the database, the cost of those extra queries will quickly dominate map generation + time. + + The tradeoff between those two approaches depends on the volume being mapped, the + speed of the disk (or SSD), the database backend being used, the number of blocks + in the database, etc. + + The worst-case behavior of this option is probably quite bad, even though it will + refuse to continue if the requested space is excessive: exceeding 1G (2^30) blocks. + Please use this option with consideration, and use `--progress`_ to monitor its + actual behavior. ``--draw[map]
" []"`` ..................................................... diff --git a/mapper.cpp b/mapper.cpp index 7641aea..257fa9a 100644 --- a/mapper.cpp +++ b/mapper.cpp @@ -39,6 +39,7 @@ using namespace std; #define OPT_DRAWHEIGHTSCALE 0x8d #define OPT_SCALEFACTOR 0x8e #define OPT_SCALEINTERVAL 0x8f +#define OPT_NO_BLOCKLIST_PREFETCH 0x90 // Will be replaced with the actual name and location of the executable (if found) string executableName = "minetestmapper"; @@ -106,6 +107,7 @@ void usage() " --min-y \n" " --max-y \n" " --backend <" USAGE_DATABASES ">\n" + " --disable-blocklist-prefetch[=force]\n" " --geometry \n" "\t(Warning: has a compatibility mode - see README.rst)\n" " --cornergeometry \n" @@ -618,6 +620,7 @@ int main(int argc, char *argv[]) {"min-y", required_argument, 0, 'a'}, {"max-y", required_argument, 0, 'c'}, {"backend", required_argument, 0, 'd'}, + {"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH}, {"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW}, {"tiles", required_argument, 0, 't'}, {"tileorigin", required_argument, 0, 'T'}, @@ -682,6 +685,20 @@ int main(int argc, char *argv[]) case 'b': generator.setBgColor(Color(optarg, 0)); break; + case OPT_NO_BLOCKLIST_PREFETCH: + if (optarg && *optarg) { + if (std::string(optarg) == "force") + generator.setGenerateNoPrefetch(2); + else { + std::cerr << "Invalid parameter to '" << long_options[option_index].name << "'; expected 'force' or nothing." << std::endl; + usage(); + exit(1); + } + } + else { + generator.setGenerateNoPrefetch(1); + } + break; case OPT_HEIGHTMAP: generator.setHeightMap(true); heightMap = true;