Add an option to specify database format (for use with --disable-blocklist-prefetch)
When using --disable-blocklist-prefetch, the database key format for every leveldb block, which is normally determined when prefetching the block list, is not known. In order to avoid duplicate queries using both key formats, the key format can now be specified using --database-format
This commit is contained in:
Rogier
parent
38f4272dd3
commit
b3ce6116a5
@ -162,6 +162,9 @@ TileGenerator::TileGenerator():
|
||||
m_heightScaleMajor(0),
|
||||
m_heightScaleMinor(0),
|
||||
m_generateNoPrefetch(0),
|
||||
m_databaseFormatSet(false),
|
||||
m_databaseFormat(BlockPos::Unknown),
|
||||
m_reportDatabaseFormat(false),
|
||||
m_image(0),
|
||||
m_xMin(INT_MAX/16-1),
|
||||
m_xMax(INT_MIN/16+1),
|
||||
@ -210,6 +213,13 @@ void TileGenerator::setGenerateNoPrefetch(int enable)
|
||||
m_generateNoPrefetch = enable;
|
||||
}
|
||||
|
||||
void TileGenerator::setDBFormat(BlockPos::StrFormat format, bool query)
|
||||
{
|
||||
m_databaseFormat = format;
|
||||
m_databaseFormatSet = true;
|
||||
m_reportDatabaseFormat = query;
|
||||
}
|
||||
|
||||
void TileGenerator::setHeightMap(bool enable)
|
||||
{
|
||||
m_heightMap = enable;
|
||||
@ -923,6 +933,17 @@ void TileGenerator::loadBlocks()
|
||||
geomYMax = MAPBLOCK_MIN;
|
||||
m_worldBlocks = 0;
|
||||
map_blocks = 0;
|
||||
if (m_reportDatabaseFormat && m_backend != "leveldb") {
|
||||
std::cerr << "WARNING: querying database format is only sensible when using the leveldb backend - querying disabled" << std::endl;
|
||||
m_reportDatabaseFormat = false;
|
||||
}
|
||||
if (m_reportDatabaseFormat && m_generateNoPrefetch) {
|
||||
std::cerr << "WARNING: querying database format cannot be combined with '--disable-blocklist-prefetch'. Prefetch disabled" << std::endl;
|
||||
m_generateNoPrefetch = false;
|
||||
}
|
||||
if (m_generateNoPrefetch && !m_databaseFormatSet && m_backend == "leveldb") {
|
||||
throw(std::runtime_error("When using --disable-blocklist-prefetch with a leveldb backend, database format must be set (--database-format)"));
|
||||
}
|
||||
if (m_generateNoPrefetch) {
|
||||
if (m_generateNoPrefetch == 1) {
|
||||
long long volume = (long long)(m_reqXMax - m_reqXMin + 1) * (m_reqYMax - m_reqYMin + 1) * (m_reqZMax - m_reqZMin + 1);
|
||||
@ -1109,8 +1130,16 @@ void TileGenerator::loadBlocks()
|
||||
<< ") blocks: "
|
||||
<< std::setw(10) << map_blocks << "\n";
|
||||
}
|
||||
if (m_backend == "leveldb") {
|
||||
if (verboseStatistics >= 3) {
|
||||
if (m_backend == "leveldb" && !m_generateNoPrefetch) {
|
||||
if (m_databaseFormatFound[BlockPos::AXYZ] && m_databaseFormatFound[BlockPos::I64])
|
||||
m_recommendedDatabaseFormat = "mixed";
|
||||
else if (m_databaseFormatFound[BlockPos::AXYZ])
|
||||
m_recommendedDatabaseFormat = "freeminer-axyz";
|
||||
else if (m_databaseFormatFound[BlockPos::I64])
|
||||
m_recommendedDatabaseFormat = "minetest-i64";
|
||||
else
|
||||
m_recommendedDatabaseFormat = "";
|
||||
if (m_reportDatabaseFormat || verboseStatistics >= 3) {
|
||||
cout
|
||||
<< std::setw(MESSAGE_WIDTH) << std::left
|
||||
<< "Database block format(s):" << std::endl
|
||||
@ -1604,11 +1633,11 @@ void TileGenerator::renderMap()
|
||||
if (m_generateNoPrefetch) {
|
||||
position = new MapBlockIteratorBlockPos();
|
||||
begin = new MapBlockIteratorBlockPos(BlockPosIterator(
|
||||
BlockPos(m_xMin, m_yMax, m_zMax),
|
||||
BlockPos(m_xMax, m_yMin, m_zMin)));
|
||||
BlockPos(m_xMin, m_yMax, m_zMax, m_databaseFormat),
|
||||
BlockPos(m_xMax, m_yMin, m_zMin, m_databaseFormat)));
|
||||
end = new MapBlockIteratorBlockPos(BlockPosIterator(
|
||||
BlockPos(m_xMin, m_yMax, m_zMax),
|
||||
BlockPos(m_xMax, m_yMin, m_zMin),
|
||||
BlockPos(m_xMin, m_yMax, m_zMax, m_databaseFormat),
|
||||
BlockPos(m_xMax, m_yMin, m_zMin, m_databaseFormat),
|
||||
BlockPosIterator::End));
|
||||
}
|
||||
else {
|
||||
@ -1724,6 +1753,15 @@ void TileGenerator::renderMap()
|
||||
<< std::setw(6) << "z"
|
||||
<< ")\n";
|
||||
}
|
||||
if (!m_generateNoPrefetch && m_backend == "leveldb" && (m_reportDatabaseFormat || verboseStatistics >= 1)) {
|
||||
cout
|
||||
<< "Database format setting when using --disable-blocklist-prefetch: ";
|
||||
if (m_recommendedDatabaseFormat != "")
|
||||
cout << m_recommendedDatabaseFormat;
|
||||
else
|
||||
cout << "unknown - use 'mixed' to be safe";
|
||||
cout << std::endl;
|
||||
}
|
||||
if (verboseStatistics >= 1) {
|
||||
eraseProgress = false;
|
||||
cout << "Statistics"
|
||||
|
||||
@ -110,6 +110,7 @@ public:
|
||||
TileGenerator();
|
||||
~TileGenerator();
|
||||
void setGenerateNoPrefetch(int enable);
|
||||
void setDBFormat(BlockPos::StrFormat format, bool query);
|
||||
void setHeightMap(bool enable);
|
||||
void setHeightMapYScale(float scale);
|
||||
void setSeaLevel(int level);
|
||||
@ -243,7 +244,11 @@ private:
|
||||
|
||||
DB *m_db;
|
||||
bool m_generateNoPrefetch;
|
||||
bool m_databaseFormatSet;
|
||||
BlockPos::StrFormat m_databaseFormat;
|
||||
std::string m_recommendedDatabaseFormat;
|
||||
long long m_databaseFormatFound[BlockPos::STRFORMAT_MAX];
|
||||
bool m_reportDatabaseFormat;
|
||||
gdImagePtr m_image;
|
||||
PixelAttributes m_blockPixelAttributes;
|
||||
PixelAttributes m_blockPixelAttributesScaled;
|
||||
|
||||
@ -289,6 +289,7 @@ Miscellaneous options
|
||||
|
||||
* ``--backend auto|sqlite3|leveldb|redis`` : Specify or override the database backend to use
|
||||
* ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds.
|
||||
* ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a leveldb backend).
|
||||
|
||||
|
||||
Detailed Description of Options
|
||||
@ -377,6 +378,46 @@ Detailed Description of Options
|
||||
|
||||
See also `--geometry`_
|
||||
|
||||
``--database-format minetest-i64|freeminer-axyz|mixed|query``
|
||||
..................................................................
|
||||
Specify the coordinate format minetest uses in the leveldb database.
|
||||
|
||||
This option is only needed, and has only effect, when
|
||||
``--disable-blocklist-prefetch`` is used, *and* when the database backend
|
||||
is 'leveldb'. Users of other backends can ignore this option.
|
||||
|
||||
A freeminer leveldb database has two possible coordinate formats. Normally,
|
||||
minetestmapper detects which one is used for which block when prefetching
|
||||
a block coordinate list.
|
||||
|
||||
With ``--disable-blocklist-prefetch``, minetestmapper will not start by reading
|
||||
a list of all blocks in the database. It therefore won't be able to detect
|
||||
what format is actually used for the coordinates of every block (which may
|
||||
differ per block).
|
||||
|
||||
Without knowing the format used for a block, the only way to be sure that it
|
||||
is not in the database, is to use two queries, one for each format. Specifying
|
||||
the format allows minetestmapper to avoid the second query, with the risk of
|
||||
overseeing blocks if they do happen to use the other format.
|
||||
|
||||
The default value for this option is ``mixed``, which works in all cases, as
|
||||
it does both queries if needed (at the very least for all blocks that are
|
||||
not in the database), but it is less efficient.
|
||||
|
||||
On minetest worlds, use ``minetest-i64``, as it is the only format used.
|
||||
|
||||
On recent freeminer worlds, use ``freeminer-axyz``, as it is the only format used.
|
||||
|
||||
``Mixed`` format is needed on older freeminer worlds, or on worlds
|
||||
that were migrated from minetest (if such worlds exist ?).
|
||||
|
||||
``Query`` directs minetestmapper to detect and report the coordinate
|
||||
format(s) used in the database. ``--disable-blocklist-prefetch`` must
|
||||
(obviously ?) be *disabled* (or will be disabled) for it to work.
|
||||
|
||||
Specifying ``minetest-i64`` or ``freeminer-axyz`` incorrectly results in all
|
||||
blocks that use the other format not being mapped.
|
||||
|
||||
``--disable-blocklist-prefetch``
|
||||
......................................
|
||||
Do not prefetch a list of block coordinates from the database before commencing
|
||||
@ -388,6 +429,9 @@ Detailed Description of Options
|
||||
It also significantly reduces the amount of information the `--verbose`_ option
|
||||
can report.
|
||||
|
||||
When used with a leveldb backend, the option `--database-format`_ should preferably
|
||||
be used as well.
|
||||
|
||||
Normally, minetestmapper will read a full list of coordinates (not the contents)
|
||||
of existing blocks from the database before starting map generation. This option
|
||||
disables this query, and instead, causes and all blocks that are in the mapped
|
||||
@ -1106,6 +1150,7 @@ Detailed Description of Options
|
||||
* maximum coordinates of the world
|
||||
* world coordinates included the map being generated
|
||||
* number of blocks: in the world, and in the map area.
|
||||
* `--database-format`_ setting if `--disable-blocklist-prefetch`_ is used.
|
||||
|
||||
Using `--verbose=2`, report some more statistics, including:
|
||||
|
||||
@ -1697,6 +1742,7 @@ More information is available:
|
||||
.. _--chunksize: `--chunksize <size>`_
|
||||
.. _--colors: `--colors <file>`_
|
||||
.. _--cornergeometry: `--cornergeometry <geometry>`_
|
||||
.. _--database-format: `--database-format minetest-i64\|freeminer-axyz\|mixed\|query`_
|
||||
.. _--draw[map]<figure>: `--draw[map]<figure> "<geometry> <color> [<text>]"`_
|
||||
.. _--draw[map]circle: `--draw[map]circle "<geometry> <color>"`_
|
||||
.. _--draw[map]ellipse: `--draw[map]ellipse "<geometry> <color>"`_
|
||||
|
||||
20
mapper.cpp
20
mapper.cpp
@ -40,6 +40,7 @@ using namespace std;
|
||||
#define OPT_SCALEFACTOR 0x8e
|
||||
#define OPT_SCALEINTERVAL 0x8f
|
||||
#define OPT_NO_BLOCKLIST_PREFETCH 0x90
|
||||
#define OPT_DATABASE_FORMAT 0x91
|
||||
|
||||
// Will be replaced with the actual name and location of the executable (if found)
|
||||
string executableName = "minetestmapper";
|
||||
@ -108,6 +109,7 @@ void usage()
|
||||
" --max-y <y>\n"
|
||||
" --backend <" USAGE_DATABASES ">\n"
|
||||
" --disable-blocklist-prefetch[=force]\n"
|
||||
" --database-format minetest-i64|freeminer-axyz|mixed|query\n"
|
||||
" --geometry <geometry>\n"
|
||||
"\t(Warning: has a compatibility mode - see README.rst)\n"
|
||||
" --cornergeometry <geometry>\n"
|
||||
@ -621,6 +623,7 @@ int main(int argc, char *argv[])
|
||||
{"max-y", required_argument, 0, 'c'},
|
||||
{"backend", required_argument, 0, 'd'},
|
||||
{"disable-blocklist-prefetch", optional_argument, 0, OPT_NO_BLOCKLIST_PREFETCH},
|
||||
{"database-format", required_argument, 0, OPT_DATABASE_FORMAT},
|
||||
{"sqlite-cacheworldrow", no_argument, 0, OPT_SQLITE_CACHEWORLDROW},
|
||||
{"tiles", required_argument, 0, 't'},
|
||||
{"tileorigin", required_argument, 0, 'T'},
|
||||
@ -699,6 +702,23 @@ int main(int argc, char *argv[])
|
||||
generator.setGenerateNoPrefetch(1);
|
||||
}
|
||||
break;
|
||||
case OPT_DATABASE_FORMAT: {
|
||||
std::string opt(optarg);
|
||||
if (opt == "minetest-i64")
|
||||
generator.setDBFormat(BlockPos::I64, false);
|
||||
else if (opt == "freeminer-axyz")
|
||||
generator.setDBFormat(BlockPos::AXYZ, false);
|
||||
else if (opt == "mixed")
|
||||
generator.setDBFormat(BlockPos::Unknown, false);
|
||||
else if (opt == "query")
|
||||
generator.setDBFormat(BlockPos::Unknown, true);
|
||||
else {
|
||||
std::cerr << "Invalid parameter to '" << long_options[option_index].name << "': '" << optarg << "'" << std::endl;
|
||||
usage();
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case OPT_HEIGHTMAP:
|
||||
generator.setHeightMap(true);
|
||||
heightMap = true;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user