Implement bounded block-list query for postgresql

i.e. allow minetestmapper to query the database for a list of
blocks in a specific range (corresponding to the requested geometry),
instead of always obtaining a full list off all blocks.

As postgresql is the only database which supports this efficiently,
this option is only effective for postgresql.
master
Rogier 2015-12-17 11:54:03 +01:00
parent 2ec0988e32
commit 03926420d0
15 changed files with 148 additions and 27 deletions

View File

@ -54,6 +54,7 @@ struct BlockPos {
bool operator<(const BlockPos& p) const;
bool operator==(const BlockPos& p) const;
bool operator!=(const BlockPos& p) const { return !operator==(p); }
void operator=(const BlockPos &p) { x() = p.x(); y() = p.y(); z() = p.z(); m_strFormat = p.m_strFormat; m_id = p.m_id; }
void operator=(int64_t i) { setFromDBPos(i); m_strFormat = I64; m_id = INT64_MIN; }
void operator=(const std::string &s);

View File

@ -11,6 +11,9 @@
in the different packages.
- Documented concurrent minetest & minetestmapper use.
- Documented the different geometries reported with --verbose
- The option --prescan-world was added to force or disable scanning the
entire world (i.e. reading a full blocklist)
(useful for postgresql, which doesn't do this by default).
Bugfixes:
- Fixed possible compilation failure caused by stdint.h
- Fixed compilation failure when some database libraries are not installed

View File

@ -110,6 +110,9 @@ static inline int readBlockContent(const unsigned char *mapData, int version, in
static const ColorEntry nodeColorNotDrawnObject;
const ColorEntry *TileGenerator::NodeColorNotDrawn = &nodeColorNotDrawnObject;
const BlockPos TileGenerator::BlockPosLimitMin(MAPBLOCK_MIN, MAPBLOCK_MIN, MAPBLOCK_MIN);
const BlockPos TileGenerator::BlockPosLimitMax(MAPBLOCK_MAX, MAPBLOCK_MAX, MAPBLOCK_MAX);
struct HeightMapColor
{
int height[2];
@ -139,6 +142,7 @@ TileGenerator::TileGenerator():
m_shading(true),
m_backend(DEFAULT_BACKEND),
m_requestedBackend(DEFAULT_BACKEND),
m_scanEntireWorld(false),
m_shrinkGeometry(true),
m_blockGeometry(false),
m_scaleFactor(1),
@ -464,6 +468,11 @@ void TileGenerator::setBackend(std::string backend)
m_requestedBackend = backend;
}
void TileGenerator::setScanEntireWorld(bool enable)
{
m_scanEntireWorld = enable;
}
void TileGenerator::setChunkSize(int size)
{
m_chunkSize = size;
@ -777,6 +786,7 @@ void TileGenerator::openDb(const std::string &input)
#if USE_SQLITE3
DBSQLite3 *db;
m_db = db = new DBSQLite3(input);
m_scanEntireWorld = true;
#else
unsupported = true;
#endif
@ -792,6 +802,7 @@ void TileGenerator::openDb(const std::string &input)
else if (m_backend == "leveldb") {
#if USE_LEVELDB
m_db = new DBLevelDB(input);
m_scanEntireWorld = true;
#else
unsupported = true;
#endif
@ -799,6 +810,7 @@ void TileGenerator::openDb(const std::string &input)
else if (m_backend == "redis") {
#if USE_REDIS
m_db = new DBRedis(input);
m_scanEntireWorld = true;
#else
unsupported = true;
#endif
@ -911,8 +923,8 @@ void TileGenerator::loadBlocks()
m_reportDatabaseFormat = false;
}
if (m_reportDatabaseFormat && m_generateNoPrefetch) {
std::cerr << "WARNING: querying database format cannot be combined with '--disable-blocklist-prefetch'. Prefetch disabled" << std::endl;
m_generateNoPrefetch = false;
std::cerr << "WARNING: querying database format: ignoring '--disable-blocklist-prefetch' and/or '--prescan-world=disabled'." << std::endl;
m_generateNoPrefetch = 0;
}
if (m_generateNoPrefetch && !m_databaseFormatSet && m_backend == "leveldb") {
throw(std::runtime_error("When using --disable-blocklist-prefetch with a leveldb backend, database format must be set (--database-format)"));
@ -922,14 +934,16 @@ void TileGenerator::loadBlocks()
long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1);
if (volume > MAX_NOPREFETCH_VOLUME) {
std::ostringstream oss;
oss << "Requested map volume is excessive for --disable-blocklist-prefetch: " << volume
// Note: the 'force' variants of the options are intentionally undocumented.
oss << "Requested map volume is excessive for --disable-blocklist-prefetch or --prescan-world=disabled: " << std::endl
<< " Volume is: " << volume
<< " (" << (m_reqXMax - m_reqXMin + 1)
<< " x " << (m_reqYMax - m_reqYMin + 1)
<< " x " << (m_reqZMax - m_reqZMin + 1)
<< " blocks of 16x16x16 nodes);"
<< "\n"
<< " Mapping will be slow. Use --disable-blocklist-prefetch='force' for more than " << MAX_NOPREFETCH_VOLUME << " blocks"
<< " (e.g. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)";
<< std::endl
<< " Mapping will be slow. Use '--disable-blocklist-prefetch=force' or '--prescan-world=disabled-force'" << std::endl
<< " to force this for more than " << MAX_NOPREFETCH_VOLUME << " blocks (i.e. " << MAX_NOPREFETCH_VOLUME_EXAMPLE << " nodes)";
throw(std::runtime_error(oss.str()));
}
}
@ -947,7 +961,16 @@ void TileGenerator::loadBlocks()
else {
if (progressIndicator)
cout << "Scanning world (reading block list)...\r" << std::flush;
const DB::BlockPosList &blocks = m_db->getBlockPos();
const DB::BlockPosList *bp;
BlockPos posMin(m_reqXMin, m_reqYMin, m_reqZMin);
BlockPos posMax(m_reqXMax, m_reqYMax, m_reqZMax);
if (!m_scanEntireWorld && (posMin != BlockPosLimitMin || posMax != BlockPosLimitMax))
bp = &m_db->getBlockPosList(BlockPos(m_reqXMin, m_reqYMin, m_reqZMin), BlockPos(m_reqXMax, m_reqYMax, m_reqZMax));
else {
m_scanEntireWorld = true;
bp = &m_db->getBlockPosList();
}
const DB::BlockPosList &blocks = *bp;
for(DB::BlockPosList::const_iterator it = blocks.begin(); it != blocks.end(); ++it) {
m_worldBlocks++;
const BlockPos &pos = *it;
@ -1003,7 +1026,7 @@ void TileGenerator::loadBlocks()
}
m_positions.push_back(pos);
}
if (verboseCoordinates >= 1) {
if (verboseCoordinates >= 1 && m_scanEntireWorld) {
if (mapXMin <= mapXMax || mapYMin <= mapYMax || mapZMin <= mapZMax) {
cout
<< std::setw(MESSAGE_WIDTH) << std::left

View File

@ -151,6 +151,7 @@ public:
void parseHeightMapNodesFile(const std::string &fileName);
void parseHeightMapColorsFile(const std::string &fileName);
void setBackend(std::string backend);
void setScanEntireWorld(bool enable);
void setChunkSize(int size);
void generate(const std::string &input, const std::string &output);
Color computeMapHeightColor(int height);
@ -215,6 +216,9 @@ private:
int linenr, const std::string &filename);
public:
static const BlockPos BlockPosLimitMin;
static const BlockPos BlockPosLimitMax;
int verboseCoordinates;
int verboseReadColors;
int verboseStatistics;
@ -239,6 +243,7 @@ private:
bool m_shading;
std::string m_backend;
std::string m_requestedBackend;
bool m_scanEntireWorld;
bool m_shrinkGeometry;
bool m_blockGeometry;
int m_scaleFactor;

View File

@ -43,7 +43,7 @@ int DBLevelDB::getBlocksQueriedCount(void)
return m_blocksQueriedCount;
}
const DB::BlockPosList &DBLevelDB::getBlockPos() {
const DB::BlockPosList &DBLevelDB::getBlockPosList() {
m_blockPosList.clear();
leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {

View File

@ -10,7 +10,7 @@ public:
DBLevelDB(const std::string &mapdir);
virtual int getBlocksQueriedCount(void);
virtual int getBlocksReadCount(void);
virtual const BlockPosList &getBlockPos();
virtual const BlockPosList &getBlockPosList();
virtual Block getBlockOnPos(const BlockPos &pos);
~DBLevelDB();
private:

View File

@ -5,8 +5,9 @@
#include "Settings.h"
#include "types.h"
#define BLOCKPOSLIST_QUERY "SELECT x, y, z FROM blocks"
#define BLOCK_QUERY "SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3"
#define BLOCKPOSLIST_QUERY "SELECT x, y, z FROM blocks"
#define BLOCKPOSLISTBOUNDED_QUERY "SELECT x, y, z FROM blocks WHERE x BETWEEN $1 AND $2 AND y BETWEEN $3 AND $4 AND z BETWEEN $5 AND $6"
#define BLOCK_QUERY "SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3"
// From pg_type.h
#define PG_INT4OID 23
@ -39,19 +40,26 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) :
}
PGresult *result;
result = PQprepare(m_connection, "GetBlockPosList", BLOCKPOSLIST_QUERY, 0, NULL);
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosList): ")
+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
PQclear(result);
result = PQprepare(m_connection, "GetBlockPosListBounded", BLOCKPOSLISTBOUNDED_QUERY, 0, NULL);
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosListBounded): ")
+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
PQclear(result);
result = PQprepare(m_connection, "GetBlock", BLOCK_QUERY, 0, NULL);
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlock): ")
+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
PQclear(result);
for (int i = 0; i < 3; i++) {
for (int i = 0; i < POSTGRESQL_MAXPARAMS; i++) {
m_getBlockParamList[i] = reinterpret_cast<char const *>(m_getBlockParams + i);
m_getBlockParamLengths[i] = sizeof(int32_t);
m_getBlockParamFormats[i] = 1;
@ -73,10 +81,25 @@ int DBPostgreSQL::getBlocksQueriedCount(void)
return m_blocksQueriedCount;
}
const DB::BlockPosList &DBPostgreSQL::getBlockPos() {
const DB::BlockPosList &DBPostgreSQL::getBlockPosList()
{
PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1);
return processBlockPosListQueryResult(result);
}
const DB::BlockPosList &DBPostgreSQL::getBlockPosList(BlockPos minPos, BlockPos maxPos)
{
for (int i = 0; i < 3; i++) {
m_getBlockParams[2*i] = htonl(minPos.dimension[i]);
m_getBlockParams[2*i+1] = htonl(maxPos.dimension[i]);
}
PGresult *result = PQexecPrepared(m_connection, "GetBlockPosListBounded", 6, m_getBlockParamList, m_getBlockParamLengths, m_getBlockParamFormats, 1);
return processBlockPosListQueryResult(result);
}
const DB::BlockPosList &DBPostgreSQL::processBlockPosListQueryResult(PGresult *result) {
m_blockPosList.clear();
PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1);
if (!result || PQresultStatus(result) != PGRES_TUPLES_OK)
throw std::runtime_error(std::string("Failed to read block-pos list from database: ")
+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));

View File

@ -23,7 +23,8 @@ public:
DBPostgreSQL(const std::string &mapdir);
virtual int getBlocksQueriedCount(void);
virtual int getBlocksReadCount(void);
virtual const BlockPosList &getBlockPos();
virtual const BlockPosList &getBlockPosList();
virtual const BlockPosList &getBlockPosList(BlockPos minPos, BlockPos maxPos);
virtual Block getBlockOnPos(const BlockPos &pos);
~DBPostgreSQL();
private:
@ -32,10 +33,13 @@ private:
PGconn *m_connection;
BlockPosList m_blockPosList;
uint32_t m_getBlockParams[3];
char const *m_getBlockParamList[3];
int m_getBlockParamLengths[3];
int m_getBlockParamFormats[3];
#define POSTGRESQL_MAXPARAMS 6
uint32_t m_getBlockParams[POSTGRESQL_MAXPARAMS];
char const *m_getBlockParamList[POSTGRESQL_MAXPARAMS];
int m_getBlockParamLengths[POSTGRESQL_MAXPARAMS];
int m_getBlockParamFormats[POSTGRESQL_MAXPARAMS];
const BlockPosList &processBlockPosListQueryResult(PGresult *result);
};
#endif // _DB_POSTGRESQL_H

View File

@ -59,7 +59,7 @@ int DBRedis::getBlocksQueriedCount(void)
}
const DB::BlockPosList &DBRedis::getBlockPos()
const DB::BlockPosList &DBRedis::getBlockPosList()
{
redisReply *reply;
reply = (redisReply*) redisCommand(ctx, "HKEYS %s", hash.c_str());

View File

@ -9,7 +9,7 @@ public:
DBRedis(const std::string &mapdir);
virtual int getBlocksQueriedCount(void);
virtual int getBlocksReadCount(void);
virtual const BlockPosList &getBlockPos();
virtual const BlockPosList &getBlockPosList();
virtual Block getBlockOnPos(const BlockPos &pos);
~DBRedis();
private:

View File

@ -48,7 +48,7 @@ int DBSQLite3::getBlocksQueriedCount(void)
return m_blocksQueriedCount;
}
const DB::BlockPosList &DBSQLite3::getBlockPos() {
const DB::BlockPosList &DBSQLite3::getBlockPosList() {
m_BlockPosList.clear();
int result = 0;
while (true) {

View File

@ -23,7 +23,7 @@ public:
DBSQLite3(const std::string &mapdir);
virtual int getBlocksQueriedCount(void);
virtual int getBlocksReadCount(void);
virtual const BlockPosList &getBlockPos();
virtual const BlockPosList &getBlockPosList();
virtual Block getBlockOnPos(const BlockPos &pos);
~DBSQLite3();
private:

3
db.h
View File

@ -14,7 +14,8 @@ class DB {
public:
typedef std::pair<BlockPos, ustring> Block;
typedef std::vector<BlockPos> BlockPosList;
virtual const BlockPosList &getBlockPos()=0;
virtual const BlockPosList &getBlockPosList()=0;
virtual const BlockPosList &getBlockPosList(BlockPos, BlockPos) { return getBlockPosList(); }
virtual int getBlocksQueriedCount(void)=0;
virtual int getBlocksReadCount(void)=0;
virtual Block getBlockOnPos(const BlockPos &pos)=0;

View File

@ -316,6 +316,7 @@ Miscellaneous options
* ``--backend auto|sqlite3|postgresql|leveldb|redis`` : Specify or override the database backend to use
* ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds.
* ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a LevelDB backend).
* ``--prescan-world=full|auto|disabled`` : Specify whether to prescan the world (compute a list of all blocks in the world).
Detailed Description of Options
@ -462,6 +463,8 @@ Detailed Description of Options
Do not prefetch a list of block coordinates from the database before commencing
map generation.
This is synonymous with `--prescan-world=disabled`_.
This option will probably improve mapping speed when mapping a smaller part
of a very large world. In other cases it may actually reduce mapping speed.
It is incompatible with, and disables, the 'shrinking' mode of `--geometrymode`_.
@ -1011,6 +1014,38 @@ Detailed Description of Options
See also `Color Syntax`_
``--prescan-world=full|auto|disabled``
........................................
Specify whether to prescan the world, i.e. whether to compute
a list of which blocks inside the area to be mapped are actually
in the database before mapping.
When ``disabled``, minetestmapper will not compute such a list at
all. While mapping, it will just attempt to load every possible
block in the section of world determined by geometry and min-y and
max-y. This is synonymous with ``--disable-blocklist-prefetch``.
See `--disable-blocklist-prefetch`_ for a discussion, caveats and
other important notes.
When set to ``full``, minetestmapper will always query the database
for the complete list of blocks which exist in the entire world. Even
if a smaller area could be queried for because of the map geometry,
min-y or max-y.
This allows the actual world dimensions to be reported, but at the
cost of additional processing time, especially if the mapped part
of the world is small compared to the existing world size.
When set to the default value: ``auto``, if possible and sensible,
minetestmapper will query the database for just a list of the blocks
in the part of the world of interested, depending on geometry,
min-y and max-y. If it does, the actual world dimensions cannot
be reported.
Unfortunately, most database backends do not support querying for a
partial block-list, or if they do, it is much less efficient than
querying for a full list. Only the PostgreSQL backend supports it
efficiently. So for all databases except PostgreSQL, ``auto`` is
equivalent to ``full``.
``--progress``
..............
@ -1207,8 +1242,10 @@ Detailed Description of Options
...................
report some useful / interesting information:
* maximum coordinates of the world
* world coordinates included the map being generated
* maximum coordinates of the world.
With a PostgreSQL backend, these are only reported if
`--prescan-world`_ is set to ``full``.
* world coordinates included the map being generated.
* number of blocks: in the world, and in the map area.
* `--database-format`_ setting if `--disable-blocklist-prefetch`_ is used.
@ -1863,6 +1900,8 @@ More information is available:
.. _--origincolor: `--origincolor <color>`_
.. _--output: `--output <output_image.png>`_
.. _--playercolor: `--playercolor <color>`_
.. _--prescan-world: `--prescan-world=full\|auto\|disabled`_
.. _--prescan-world=disabled: `--prescan-world=full\|auto\|disabled`_
.. _--silence-suggestions: `--silence-suggestions all,prefetch`_
.. _--scalecolor: `--scalecolor <color>`_
.. _--scalefactor: `--scalefactor 1:<n>`_

View File

@ -21,6 +21,7 @@
#include <sys/types.h>
#include "TileGenerator.h"
#include "PixelAttributes.h"
#include "db-postgresql.h"
using namespace std;
@ -42,6 +43,7 @@ using namespace std;
#define OPT_NO_BLOCKLIST_PREFETCH 0x90
#define OPT_DATABASE_FORMAT 0x91
#define OPT_SILENCE_SUGGESTIONS 0x92
#define OPT_PRESCAN_WORLD 0x93
// Will be replaced with the actual name and location of the executable (if found)
string executableName = "minetestmapper";
@ -131,6 +133,7 @@ void usage()
" --backend <" USAGE_DATABASES ">\n"
" --disable-blocklist-prefetch[=force]\n"
" --database-format minetest-i64|freeminer-axyz|mixed|query\n"
" --prescan-world=full|auto|disabled\n"
" --geometry <geometry>\n"
"\t(Warning: has a compatibility mode - see README.rst)\n"
" --cornergeometry <geometry>\n"
@ -646,6 +649,7 @@ int main(int argc, char *argv[])
{"backend", required_argument, 0, 'd'},
{"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH},
{"database-format", required_argument, 0, OPT_DATABASE_FORMAT},
{"prescan-world", required_argument, 0, OPT_PRESCAN_WORLD},
{"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW},
{"tiles", required_argument, 0, 't'},
{"tileorigin", required_argument, 0, 'T'},
@ -742,6 +746,24 @@ int main(int argc, char *argv[])
}
}
break;
case OPT_PRESCAN_WORLD: {
std::string opt(optarg);
generator.setGenerateNoPrefetch(0);
if (opt == "disabled-force")
generator.setGenerateNoPrefetch(2);
else if (opt == "disabled")
generator.setGenerateNoPrefetch(1);
else if (opt == "auto")
generator.setScanEntireWorld(false);
else if (opt == "full")
generator.setScanEntireWorld(true);
else {
std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': '" << optarg << "'" << std::endl;
usage();
exit(1);
}
}
break;
case OPT_HEIGHTMAP:
generator.setHeightMap(true);
heightMap = true;