diff --git a/BlockPos.h b/BlockPos.h index 352cd8d..183937d 100644 --- a/BlockPos.h +++ b/BlockPos.h @@ -54,6 +54,7 @@ struct BlockPos { bool operator<(const BlockPos& p) const; bool operator==(const BlockPos& p) const; + bool operator!=(const BlockPos& p) const { return !operator==(p); } void operator=(const BlockPos &p) { x() = p.x(); y() = p.y(); z() = p.z(); m_strFormat = p.m_strFormat; m_id = p.m_id; } void operator=(int64_t i) { setFromDBPos(i); m_strFormat = I64; m_id = INT64_MIN; } void operator=(const std::string &s); diff --git a/Changelog b/Changelog index 9a42191..209408c 100644 --- a/Changelog +++ b/Changelog @@ -11,6 +11,9 @@ in the different packages. - Documented concurrent minetest & minetestmapper use. - Documented the different geometries reported with --verbose + - The option --prescan-world was added to force or disable scanning the + entire world (i.e. reading a full blocklist) + (useful for postgresql, which doesn't do this by default). Bugfixes: - Fixed possible compilation failure caused by stdint.h - Fixed compilation failure when some database libraries are not installed diff --git a/TileGenerator.cpp b/TileGenerator.cpp index 40f43b9..de03c36 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -110,6 +110,9 @@ static inline int readBlockContent(const unsigned char *mapData, int version, in static const ColorEntry nodeColorNotDrawnObject; const ColorEntry *TileGenerator::NodeColorNotDrawn = &nodeColorNotDrawnObject; +const BlockPos TileGenerator::BlockPosLimitMin(MAPBLOCK_MIN, MAPBLOCK_MIN, MAPBLOCK_MIN); +const BlockPos TileGenerator::BlockPosLimitMax(MAPBLOCK_MAX, MAPBLOCK_MAX, MAPBLOCK_MAX); + struct HeightMapColor { int height[2]; @@ -139,6 +142,7 @@ TileGenerator::TileGenerator(): m_shading(true), m_backend(DEFAULT_BACKEND), m_requestedBackend(DEFAULT_BACKEND), + m_scanEntireWorld(false), m_shrinkGeometry(true), m_blockGeometry(false), m_scaleFactor(1), @@ -464,6 +468,11 @@ void TileGenerator::setBackend(std::string backend) m_requestedBackend = backend; } +void TileGenerator::setScanEntireWorld(bool enable) +{ + m_scanEntireWorld = enable; +} + void TileGenerator::setChunkSize(int size) { m_chunkSize = size; @@ -777,6 +786,7 @@ void TileGenerator::openDb(const std::string &input) #if USE_SQLITE3 DBSQLite3 *db; m_db = db = new DBSQLite3(input); + m_scanEntireWorld = true; #else unsupported = true; #endif @@ -792,6 +802,7 @@ void TileGenerator::openDb(const std::string &input) else if (m_backend == "leveldb") { #if USE_LEVELDB m_db = new DBLevelDB(input); + m_scanEntireWorld = true; #else unsupported = true; #endif @@ -799,6 +810,7 @@ void TileGenerator::openDb(const std::string &input) else if (m_backend == "redis") { #if USE_REDIS m_db = new DBRedis(input); + m_scanEntireWorld = true; #else unsupported = true; #endif @@ -911,8 +923,8 @@ void TileGenerator::loadBlocks() m_reportDatabaseFormat = false; } if (m_reportDatabaseFormat && m_generateNoPrefetch) { - std::cerr << "WARNING: querying database format cannot be combined with '--disable-blocklist-prefetch'. Prefetch disabled" << std::endl; - m_generateNoPrefetch = false; + std::cerr << "WARNING: querying database format: ignoring '--disable-blocklist-prefetch' and/or '--prescan-world=disabled'." << std::endl; + m_generateNoPrefetch = 0; } if (m_generateNoPrefetch && !m_databaseFormatSet && m_backend == "leveldb") { throw(std::runtime_error("When using --disable-blocklist-prefetch with a leveldb backend, database format must be set (--database-format)")); @@ -922,14 +934,16 @@ void TileGenerator::loadBlocks() long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1); if (volume > MAX_NOPREFETCH_VOLUME) { std::ostringstream oss; - oss << "Requested map volume is excessive for --disable-blocklist-prefetch: " << volume + // Note: the 'force' variants of the options are intentionally undocumented. + oss << "Requested map volume is excessive for --disable-blocklist-prefetch or --prescan-world=disabled: " << std::endl + << " Volume is: " << volume << " (" << (m_reqXMax - m_reqXMin + 1) << " x " << (m_reqYMax - m_reqYMin + 1) << " x " << (m_reqZMax - m_reqZMin + 1) << " blocks of 16x16x16 nodes);" - << "\n" - << " Mapping will be slow. Use --disable-blocklist-prefetch='force' for more than " << MAX_NOPREFETCH_VOLUME << " blocks" - << " (e.g. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)"; + << std::endl + << " Mapping will be slow. Use '--disable-blocklist-prefetch=force' or '--prescan-world=disabled-force'" << std::endl + << " to force this for more than " << MAX_NOPREFETCH_VOLUME << " blocks (i.e. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)"; throw(std::runtime_error(oss.str())); } } @@ -947,7 +961,16 @@ void TileGenerator::loadBlocks() else { if (progressIndicator) cout << "Scanning world (reading block list)...\r" << std::flush; - const DB::BlockPosList &blocks = m_db->getBlockPos(); + const DB::BlockPosList *bp; + BlockPos posMin(m_reqXMin, m_reqYMin, m_reqZMin); + BlockPos posMax(m_reqXMax, m_reqYMax, m_reqZMax); + if (!m_scanEntireWorld && (posMin != BlockPosLimitMin || posMax != BlockPosLimitMax)) + bp = &m_db->getBlockPosList(BlockPos(m_reqXMin, m_reqYMin, m_reqZMin), BlockPos(m_reqXMax, m_reqYMax, m_reqZMax)); + else { + m_scanEntireWorld = true; + bp = &m_db->getBlockPosList(); + } + const DB::BlockPosList &blocks = *bp; for(DB::BlockPosList::const_iterator it = blocks.begin(); it != blocks.end(); ++it) { m_worldBlocks++; const BlockPos &pos = *it; @@ -1003,7 +1026,7 @@ void TileGenerator::loadBlocks() } m_positions.push_back(pos); } - if (verboseCoordinates >= 1) { + if (verboseCoordinates >= 1 && m_scanEntireWorld) { if (mapXMin <= mapXMax || mapYMin <= mapYMax || mapZMin <= mapZMax) { cout << std::setw(MESSAGE_WIDTH) << std::left diff --git a/TileGenerator.h b/TileGenerator.h index e49323e..6959370 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -151,6 +151,7 @@ public: void parseHeightMapNodesFile(const std::string &fileName); void parseHeightMapColorsFile(const std::string &fileName); void setBackend(std::string backend); + void setScanEntireWorld(bool enable); void setChunkSize(int size); void generate(const std::string &input, const std::string &output); Color computeMapHeightColor(int height); @@ -215,6 +216,9 @@ private: int linenr, const std::string &filename); public: + static const BlockPos BlockPosLimitMin; + static const BlockPos BlockPosLimitMax; + int verboseCoordinates; int verboseReadColors; int verboseStatistics; @@ -239,6 +243,7 @@ private: bool m_shading; std::string m_backend; std::string m_requestedBackend; + bool m_scanEntireWorld; bool m_shrinkGeometry; bool m_blockGeometry; int m_scaleFactor; diff --git a/db-leveldb.cpp b/db-leveldb.cpp index 730ee21..9492c2f 100644 --- a/db-leveldb.cpp +++ b/db-leveldb.cpp @@ -43,7 +43,7 @@ int DBLevelDB::getBlocksQueriedCount(void) return m_blocksQueriedCount; } -const DB::BlockPosList &DBLevelDB::getBlockPos() { +const DB::BlockPosList &DBLevelDB::getBlockPosList() { m_blockPosList.clear(); leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { diff --git a/db-leveldb.h b/db-leveldb.h index bc2ddbb..d16afce 100644 --- a/db-leveldb.h +++ b/db-leveldb.h @@ -10,7 +10,7 @@ public: DBLevelDB(const std::string &mapdir); virtual int getBlocksQueriedCount(void); virtual int getBlocksReadCount(void); - virtual const BlockPosList &getBlockPos(); + virtual const BlockPosList &getBlockPosList(); virtual Block getBlockOnPos(const BlockPos &pos); ~DBLevelDB(); private: diff --git a/db-postgresql.cpp b/db-postgresql.cpp index 95bf5b8..15d5300 100644 --- a/db-postgresql.cpp +++ b/db-postgresql.cpp @@ -5,8 +5,9 @@ #include "Settings.h" #include "types.h" -#define BLOCKPOSLIST_QUERY "SELECT x, y, z FROM blocks" -#define BLOCK_QUERY "SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3" +#define BLOCKPOSLIST_QUERY "SELECT x, y, z FROM blocks" +#define BLOCKPOSLISTBOUNDED_QUERY "SELECT x, y, z FROM blocks WHERE x BETWEEN $1 AND $2 AND y BETWEEN $3 AND $4 AND z BETWEEN $5 AND $6" +#define BLOCK_QUERY "SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3" // From pg_type.h #define PG_INT4OID 23 @@ -39,19 +40,26 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) : } PGresult *result; + result = PQprepare(m_connection, "GetBlockPosList", BLOCKPOSLIST_QUERY, 0, NULL); if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosList): ") + (result ? PQresultErrorMessage(result) : "(result was NULL)")); PQclear(result); + result = PQprepare(m_connection, "GetBlockPosListBounded", BLOCKPOSLISTBOUNDED_QUERY, 0, NULL); + if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) + throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosListBounded): ") + + (result ? PQresultErrorMessage(result) : "(result was NULL)")); + PQclear(result); + result = PQprepare(m_connection, "GetBlock", BLOCK_QUERY, 0, NULL); if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlock): ") + (result ? PQresultErrorMessage(result) : "(result was NULL)")); PQclear(result); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < POSTGRESQL_MAXPARAMS; i++) { m_getBlockParamList[i] = reinterpret_cast(m_getBlockParams + i); m_getBlockParamLengths[i] = sizeof(int32_t); m_getBlockParamFormats[i] = 1; @@ -73,10 +81,25 @@ int DBPostgreSQL::getBlocksQueriedCount(void) return m_blocksQueriedCount; } -const DB::BlockPosList &DBPostgreSQL::getBlockPos() { +const DB::BlockPosList &DBPostgreSQL::getBlockPosList() +{ + PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1); + return processBlockPosListQueryResult(result); +} + +const DB::BlockPosList &DBPostgreSQL::getBlockPosList(BlockPos minPos, BlockPos maxPos) +{ + for (int i = 0; i < 3; i++) { + m_getBlockParams[2*i] = htonl(minPos.dimension[i]); + m_getBlockParams[2*i+1] = htonl(maxPos.dimension[i]); + } + PGresult *result = PQexecPrepared(m_connection, "GetBlockPosListBounded", 6, m_getBlockParamList, m_getBlockParamLengths, m_getBlockParamFormats, 1); + return processBlockPosListQueryResult(result); +} + +const DB::BlockPosList &DBPostgreSQL::processBlockPosListQueryResult(PGresult *result) { m_blockPosList.clear(); - PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1); if (!result || PQresultStatus(result) != PGRES_TUPLES_OK) throw std::runtime_error(std::string("Failed to read block-pos list from database: ") + (result ? PQresultErrorMessage(result) : "(result was NULL)")); diff --git a/db-postgresql.h b/db-postgresql.h index 79d10e0..86a2c6d 100644 --- a/db-postgresql.h +++ b/db-postgresql.h @@ -23,7 +23,8 @@ public: DBPostgreSQL(const std::string &mapdir); virtual int getBlocksQueriedCount(void); virtual int getBlocksReadCount(void); - virtual const BlockPosList &getBlockPos(); + virtual const BlockPosList &getBlockPosList(); + virtual const BlockPosList &getBlockPosList(BlockPos minPos, BlockPos maxPos); virtual Block getBlockOnPos(const BlockPos &pos); ~DBPostgreSQL(); private: @@ -32,10 +33,13 @@ private: PGconn *m_connection; BlockPosList m_blockPosList; - uint32_t m_getBlockParams[3]; - char const *m_getBlockParamList[3]; - int m_getBlockParamLengths[3]; - int m_getBlockParamFormats[3]; + #define POSTGRESQL_MAXPARAMS 6 + uint32_t m_getBlockParams[POSTGRESQL_MAXPARAMS]; + char const *m_getBlockParamList[POSTGRESQL_MAXPARAMS]; + int m_getBlockParamLengths[POSTGRESQL_MAXPARAMS]; + int m_getBlockParamFormats[POSTGRESQL_MAXPARAMS]; + + const BlockPosList &processBlockPosListQueryResult(PGresult *result); }; #endif // _DB_POSTGRESQL_H diff --git a/db-redis.cpp b/db-redis.cpp index 213a81b..d63e1a4 100644 --- a/db-redis.cpp +++ b/db-redis.cpp @@ -59,7 +59,7 @@ int DBRedis::getBlocksQueriedCount(void) } -const DB::BlockPosList &DBRedis::getBlockPos() +const DB::BlockPosList &DBRedis::getBlockPosList() { redisReply *reply; reply = (redisReply*) redisCommand(ctx, "HKEYS %s", hash.c_str()); diff --git a/db-redis.h b/db-redis.h index dea1c9b..0eb5094 100644 --- a/db-redis.h +++ b/db-redis.h @@ -9,7 +9,7 @@ public: DBRedis(const std::string &mapdir); virtual int getBlocksQueriedCount(void); virtual int getBlocksReadCount(void); - virtual const BlockPosList &getBlockPos(); + virtual const BlockPosList &getBlockPosList(); virtual Block getBlockOnPos(const BlockPos &pos); ~DBRedis(); private: diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index a427d21..552395d 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -48,7 +48,7 @@ int DBSQLite3::getBlocksQueriedCount(void) return m_blocksQueriedCount; } -const DB::BlockPosList &DBSQLite3::getBlockPos() { +const DB::BlockPosList &DBSQLite3::getBlockPosList() { m_BlockPosList.clear(); int result = 0; while (true) { diff --git a/db-sqlite3.h b/db-sqlite3.h index 29f6918..68205f2 100644 --- a/db-sqlite3.h +++ b/db-sqlite3.h @@ -23,7 +23,7 @@ public: DBSQLite3(const std::string &mapdir); virtual int getBlocksQueriedCount(void); virtual int getBlocksReadCount(void); - virtual const BlockPosList &getBlockPos(); + virtual const BlockPosList &getBlockPosList(); virtual Block getBlockOnPos(const BlockPos &pos); ~DBSQLite3(); private: diff --git a/db.h b/db.h index 4c74876..c9f06c8 100644 --- a/db.h +++ b/db.h @@ -14,7 +14,8 @@ class DB { public: typedef std::pair Block; typedef std::vector BlockPosList; - virtual const BlockPosList &getBlockPos()=0; + virtual const BlockPosList &getBlockPosList()=0; + virtual const BlockPosList &getBlockPosList(BlockPos, BlockPos) { return getBlockPosList(); } virtual int getBlocksQueriedCount(void)=0; virtual int getBlocksReadCount(void)=0; virtual Block getBlockOnPos(const BlockPos &pos)=0; diff --git a/doc/manual.rst b/doc/manual.rst index 8284c57..28b0d22 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -316,6 +316,7 @@ Miscellaneous options * ``--backend auto|sqlite3|postgresql|leveldb|redis`` : Specify or override the database backend to use * ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds. * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a LevelDB backend). + * ``--prescan-world=full|auto|disabled`` : Specify whether to prescan the world (compute a list of all blocks in the world). Detailed Description of Options @@ -462,6 +463,8 @@ Detailed Description of Options Do not prefetch a list of block coordinates from the database before commencing map generation. + This is synonymous with `--prescan-world=disabled`_. + This option will probably improve mapping speed when mapping a smaller part of a very large world. In other cases it may actually reduce mapping speed. It is incompatible with, and disables, the 'shrinking' mode of `--geometrymode`_. @@ -1011,6 +1014,38 @@ Detailed Description of Options See also `Color Syntax`_ +``--prescan-world=full|auto|disabled`` +........................................ + Specify whether to prescan the world, i.e. whether to compute + a list of which blocks inside the area to be mapped are actually + in the database before mapping. + + When ``disabled``, minetestmapper will not compute such a list at + all. While mapping, it will just attempt to load every possible + block in the section of world determined by geometry and min-y and + max-y. This is synonymous with ``--disable-blocklist-prefetch``. + See `--disable-blocklist-prefetch`_ for a discussion, caveats and + other important notes. + + When set to ``full``, minetestmapper will always query the database + for the complete list of blocks which exist in the entire world. Even + if a smaller area could be queried for because of the map geometry, + min-y or max-y. + This allows the actual world dimensions to be reported, but at the + cost of additional processing time, especially if the mapped part + of the world is small compared to the existing world size. + + When set to the default value: ``auto``, if possible and sensible, + minetestmapper will query the database for just a list of the blocks + in the part of the world of interested, depending on geometry, + min-y and max-y. If it does, the actual world dimensions cannot + be reported. + + Unfortunately, most database backends do not support querying for a + partial block-list, or if they do, it is much less efficient than + querying for a full list. Only the PostgreSQL backend supports it + efficiently. So for all databases except PostgreSQL, ``auto`` is + equivalent to ``full``. ``--progress`` .............. @@ -1207,8 +1242,10 @@ Detailed Description of Options ................... report some useful / interesting information: - * maximum coordinates of the world - * world coordinates included the map being generated + * maximum coordinates of the world. + With a PostgreSQL backend, these are only reported if + `--prescan-world`_ is set to ``full``. + * world coordinates included the map being generated. * number of blocks: in the world, and in the map area. * `--database-format`_ setting if `--disable-blocklist-prefetch`_ is used. @@ -1863,6 +1900,8 @@ More information is available: .. _--origincolor: `--origincolor `_ .. _--output: `--output `_ .. _--playercolor: `--playercolor `_ +.. _--prescan-world: `--prescan-world=full\|auto\|disabled`_ +.. _--prescan-world=disabled: `--prescan-world=full\|auto\|disabled`_ .. _--silence-suggestions: `--silence-suggestions all,prefetch`_ .. _--scalecolor: `--scalecolor `_ .. _--scalefactor: `--scalefactor 1:`_ diff --git a/mapper.cpp b/mapper.cpp index dbdf4a0..3528da7 100644 --- a/mapper.cpp +++ b/mapper.cpp @@ -21,6 +21,7 @@ #include #include "TileGenerator.h" #include "PixelAttributes.h" +#include "db-postgresql.h" using namespace std; @@ -42,6 +43,7 @@ using namespace std; #define OPT_NO_BLOCKLIST_PREFETCH 0x90 #define OPT_DATABASE_FORMAT 0x91 #define OPT_SILENCE_SUGGESTIONS 0x92 +#define OPT_PRESCAN_WORLD 0x93 // Will be replaced with the actual name and location of the executable (if found) string executableName = "minetestmapper"; @@ -131,6 +133,7 @@ void usage() " --backend <" USAGE_DATABASES ">\n" " --disable-blocklist-prefetch[=force]\n" " --database-format minetest-i64|freeminer-axyz|mixed|query\n" + " --prescan-world=full|auto|disabled\n" " --geometry \n" "\t(Warning: has a compatibility mode - see README.rst)\n" " --cornergeometry \n" @@ -646,6 +649,7 @@ int main(int argc, char *argv[]) {"backend", required_argument, 0, 'd'}, {"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH}, {"database-format", required_argument, 0, OPT_DATABASE_FORMAT}, + {"prescan-world", required_argument, 0, OPT_PRESCAN_WORLD}, {"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW}, {"tiles", required_argument, 0, 't'}, {"tileorigin", required_argument, 0, 'T'}, @@ -742,6 +746,24 @@ int main(int argc, char *argv[]) } } break; + case OPT_PRESCAN_WORLD: { + std::string opt(optarg); + generator.setGenerateNoPrefetch(0); + if (opt == "disabled-force") + generator.setGenerateNoPrefetch(2); + else if (opt == "disabled") + generator.setGenerateNoPrefetch(1); + else if (opt == "auto") + generator.setScanEntireWorld(false); + else if (opt == "full") + generator.setScanEntireWorld(true); + else { + std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': '" << optarg << "'" << std::endl; + usage(); + exit(1); + } + } + break; case OPT_HEIGHTMAP: generator.setHeightMap(true); heightMap = true;