From b3ce6116a53df953deb8958302b152e3d5d25333 Mon Sep 17 00:00:00 2001 From: Rogier Date: Tue, 10 Mar 2015 13:16:43 +0100 Subject: [PATCH] Add an option to specify database format (for use with --disable-blocklist-prefetch) When using --disable-blocklist-prefetch, the database key format for every leveldb block, which is normally determined when prefetching the block list, is not known. In order to avoid duplicate queries using both key formats, the key format can now be specified using --database-format --- TileGenerator.cpp | 50 +++++++++++++++++++++++++++++++++++++++++------ TileGenerator.h | 5 +++++ doc/manual.rst | 46 +++++++++++++++++++++++++++++++++++++++++++ mapper.cpp | 20 +++++++++++++++++++ 4 files changed, 115 insertions(+), 6 deletions(-) diff --git a/TileGenerator.cpp b/TileGenerator.cpp index 7e61c27..43c7e87 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -162,6 +162,9 @@ TileGenerator::TileGenerator(): m_heightScaleMajor(0), m_heightScaleMinor(0), m_generateNoPrefetch(0), + m_databaseFormatSet(false), + m_databaseFormat(BlockPos::Unknown), + m_reportDatabaseFormat(false), m_image(0), m_xMin(INT_MAX/16-1), m_xMax(INT_MIN/16+1), @@ -210,6 +213,13 @@ void TileGenerator::setGenerateNoPrefetch(int enable) m_generateNoPrefetch = enable; } +void TileGenerator::setDBFormat(BlockPos::StrFormat format, bool query) +{ + m_databaseFormat = format; + m_databaseFormatSet = true; + m_reportDatabaseFormat = query; +} + void TileGenerator::setHeightMap(bool enable) { m_heightMap = enable; @@ -923,6 +933,17 @@ void TileGenerator::loadBlocks() geomYMax = MAPBLOCK_MIN; m_worldBlocks = 0; map_blocks = 0; + if (m_reportDatabaseFormat && m_backend != "leveldb") { + std::cerr << "WARNING: querying database format is only sensible when using the leveldb backend - querying disabled" << std::endl; + m_reportDatabaseFormat = false; + } + if (m_reportDatabaseFormat && m_generateNoPrefetch) { + std::cerr << "WARNING: querying database format cannot be combined with '--disable-blocklist-prefetch'. Prefetch disabled" << std::endl; + m_generateNoPrefetch = false; + } + if (m_generateNoPrefetch && !m_databaseFormatSet && m_backend == "leveldb") { + throw(std::runtime_error("When using --disable-blocklist-prefetch with a leveldb backend, database format must be set (--database-format)")); + } if (m_generateNoPrefetch) { if (m_generateNoPrefetch == 1) { long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1); @@ -1109,8 +1130,16 @@ void TileGenerator::loadBlocks() << ") blocks: " << std::setw(10) << map_blocks << "\n"; } - if (m_backend == "leveldb") { - if (verboseStatistics >= 3) { + if (m_backend == "leveldb" && !m_generateNoPrefetch) { + if (m_databaseFormatFound[BlockPos::AXYZ] && m_databaseFormatFound[BlockPos::I64]) + m_recommendedDatabaseFormat = "mixed"; + else if (m_databaseFormatFound[BlockPos::AXYZ]) + m_recommendedDatabaseFormat = "freeminer-axyz"; + else if (m_databaseFormatFound[BlockPos::I64]) + m_recommendedDatabaseFormat = "minetest-i64"; + else + m_recommendedDatabaseFormat = ""; + if (m_reportDatabaseFormat || verboseStatistics >= 3) { cout << std::setw(MESSAGE_WIDTH) << std::left << "Database block format(s):" << std::endl @@ -1604,11 +1633,11 @@ void TileGenerator::renderMap() if (m_generateNoPrefetch) { position = new MapBlockIteratorBlockPos(); begin = new MapBlockIteratorBlockPos(BlockPosIterator( - BlockPos(m_xMin, m_yMax, m_zMax), - BlockPos(m_xMax, m_yMin, m_zMin))); + BlockPos(m_xMin, m_yMax, m_zMax, m_databaseFormat), + BlockPos(m_xMax, m_yMin, m_zMin, m_databaseFormat))); end = new MapBlockIteratorBlockPos(BlockPosIterator( - BlockPos(m_xMin, m_yMax, m_zMax), - BlockPos(m_xMax, m_yMin, m_zMin), + BlockPos(m_xMin, m_yMax, m_zMax, m_databaseFormat), + BlockPos(m_xMax, m_yMin, m_zMin, m_databaseFormat), BlockPosIterator::End)); } else { @@ -1724,6 +1753,15 @@ void TileGenerator::renderMap() << std::setw(6) << "z" << ")\n"; } + if (!m_generateNoPrefetch && m_backend == "leveldb" && (m_reportDatabaseFormat || verboseStatistics >= 1)) { + cout + << "Database format setting when using --disable-blocklist-prefetch: "; + if (m_recommendedDatabaseFormat != "") + cout << m_recommendedDatabaseFormat; + else + cout << "unknown - use 'mixed' to be safe"; + cout << std::endl; + } if (verboseStatistics >= 1) { eraseProgress = false; cout << "Statistics" diff --git a/TileGenerator.h b/TileGenerator.h index f353ad7..3c19000 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -110,6 +110,7 @@ public: TileGenerator(); ~TileGenerator(); void setGenerateNoPrefetch(int enable); + void setDBFormat(BlockPos::StrFormat format, bool query); void setHeightMap(bool enable); void setHeightMapYScale(float scale); void setSeaLevel(int level); @@ -243,7 +244,11 @@ private: DB *m_db; bool m_generateNoPrefetch; + bool m_databaseFormatSet; + BlockPos::StrFormat m_databaseFormat; + std::string m_recommendedDatabaseFormat; long long m_databaseFormatFound[BlockPos::STRFORMAT_MAX]; + bool m_reportDatabaseFormat; gdImagePtr m_image; PixelAttributes m_blockPixelAttributes; PixelAttributes m_blockPixelAttributesScaled; diff --git a/doc/manual.rst b/doc/manual.rst index 6dfc7f3..958f095 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -289,6 +289,7 @@ Miscellaneous options * ``--backend auto|sqlite3|leveldb|redis`` : Specify or override the database backend to use * ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds. + * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a leveldb backend). Detailed Description of Options @@ -377,6 +378,46 @@ Detailed Description of Options See also `--geometry`_ +``--database-format minetest-i64|freeminer-axyz|mixed|query`` +.................................................................. + Specify the coordinate format minetest uses in the leveldb database. + + This option is only needed, and has only effect, when + ``--disable-blocklist-prefetch`` is used, *and* when the database backend + is 'leveldb'. Users of other backends can ignore this option. + + A freeminer leveldb database has two possible coordinate formats. Normally, + minetestmapper detects which one is used for which block when prefetching + a block coordinate list. + + With ``--disable-blocklist-prefetch``, minetestmapper will not start by reading + a list of all blocks in the database. It therefore won't be able to detect + what format is actually used for the coordinates of every block (which may + differ per block). + + Without knowing the format used for a block, the only way to be sure that it + is not in the database, is to use two queries, one for each format. Specifying + the format allows minetestmapper to avoid the second query, with the risk of + overseeing blocks if they do happen to use the other format. + + The default value for this option is ``mixed``, which works in all cases, as + it does both queries if needed (at the very least for all blocks that are + not in the database), but it is less efficient. + + On minetest worlds, use ``minetest-i64``, as it is the only format used. + + On recent freeminer worlds, use ``freeminer-axyz``, as it is the only format used. + + ``Mixed`` format is needed on older freeminer worlds, or on worlds + that were migrated from minetest (if such worlds exist ?). + + ``Query`` directs minetestmapper to detect and report the coordinate + format(s) used in the database. ``--disable-blocklist-prefetch`` must + (obviously ?) be *disabled* (or will be disabled) for it to work. + + Specifying ``minetest-i64`` or ``freeminer-axyz`` incorrectly results in all + blocks that use the other format not being mapped. + ``--disable-blocklist-prefetch`` ...................................... Do not prefetch a list of block coordinates from the database before commencing @@ -388,6 +429,9 @@ Detailed Description of Options It also significantly reduces the amount of information the `--verbose`_ option can report. + When used with a leveldb backend, the option `--database-format`_ should preferably + be used as well. + Normally, minetestmapper will read a full list of coordinates (not the contents) of existing blocks from the database before starting map generation. This option disables this query, and instead, causes and all blocks that are in the mapped @@ -1106,6 +1150,7 @@ Detailed Description of Options * maximum coordinates of the world * world coordinates included the map being generated * number of blocks: in the world, and in the map area. + * `--database-format`_ setting if `--disable-blocklist-prefetch`_ is used. Using `--verbose=2`, report some more statistics, including: @@ -1697,6 +1742,7 @@ More information is available: .. _--chunksize: `--chunksize `_ .. _--colors: `--colors `_ .. _--cornergeometry: `--cornergeometry `_ +.. _--database-format: `--database-format minetest-i64\|freeminer-axyz\|mixed\|query`_ .. _--draw[map]
: `--draw[map]
" []"`_ .. _--draw[map]circle: `--draw[map]circle " "`_ .. _--draw[map]ellipse: `--draw[map]ellipse " "`_ diff --git a/mapper.cpp b/mapper.cpp index 257fa9a..7cff181 100644 --- a/mapper.cpp +++ b/mapper.cpp @@ -40,6 +40,7 @@ using namespace std; #define OPT_SCALEFACTOR 0x8e #define OPT_SCALEINTERVAL 0x8f #define OPT_NO_BLOCKLIST_PREFETCH 0x90 +#define OPT_DATABASE_FORMAT 0x91 // Will be replaced with the actual name and location of the executable (if found) string executableName = "minetestmapper"; @@ -108,6 +109,7 @@ void usage() " --max-y \n" " --backend <" USAGE_DATABASES ">\n" " --disable-blocklist-prefetch[=force]\n" + " --database-format minetest-i64|freeminer-axyz|mixed|query\n" " --geometry \n" "\t(Warning: has a compatibility mode - see README.rst)\n" " --cornergeometry \n" @@ -621,6 +623,7 @@ int main(int argc, char *argv[]) {"max-y", required_argument, 0, 'c'}, {"backend", required_argument, 0, 'd'}, {"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH}, + {"database-format", required_argument, 0, OPT_DATABASE_FORMAT}, {"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW}, {"tiles", required_argument, 0, 't'}, {"tileorigin", required_argument, 0, 'T'}, @@ -699,6 +702,23 @@ int main(int argc, char *argv[]) generator.setGenerateNoPrefetch(1); } break; + case OPT_DATABASE_FORMAT: { + std::string opt(optarg); + if (opt == "minetest-i64") + generator.setDBFormat(BlockPos::I64, false); + else if (opt == "freeminer-axyz") + generator.setDBFormat(BlockPos::AXYZ, false); + else if (opt == "mixed") + generator.setDBFormat(BlockPos::Unknown, false); + else if (opt == "query") + generator.setDBFormat(BlockPos::Unknown, true); + else { + std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': '" << optarg << "'" << std::endl; + usage(); + exit(1); + } + } + break; case OPT_HEIGHTMAP: generator.setHeightMap(true); heightMap = true;