diff --git a/CMakeLists.txt b/CMakeLists.txt index 3970913..35ad76b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,12 +144,14 @@ include(FindPackageHandleStandardArgs) # Find database(s) set(USE_SQLITE3 0) +set(USE_POSTGRESQL 0) set(USE_LEVELDB 0) set(USE_REDIS 0) OPTION(ENABLE_ANY_DATABASE "Enable any available database backends" True) OPTION(ENABLE_ALL_DATABASES "Enable all possible database backends") OPTION(ENABLE_SQLITE3 "Enable sqlite3 backend" True) +OPTION(ENABLE_POSTGRESQL "Enable postgresql backend") OPTION(ENABLE_LEVELDB "Enable LevelDB backend") OPTION(ENABLE_REDIS "Enable redis backend") @@ -173,6 +175,26 @@ if(ENABLE_SQLITE3 OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) endif(SQLITE3_LIBRARY AND SQLITE3_INCLUDE_DIR) endif(ENABLE_SQLITE3 OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) +# Find postgresql +if(ENABLE_POSTGRESQL OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) + find_library(POSTGRESQL_LIBRARY pq) + find_path(POSTGRESQL_INCLUDE_DIR libpq-fe.h PATH_SUFFIXES postgresql) + message (STATUS "postgresql library: ${POSTGRESQL_LIBRARY}") + message (STATUS "postgresql headers: ${POSTGRESQL_INCLUDE_DIR}") + if(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR) + set(USE_POSTGRESQL 1) + message(STATUS "postgresql backend enabled") + include_directories(${POSTGRESQL_INCLUDE_DIR}) + else(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR) + set(USE_POSTGRESQL 0) + if(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES) + message(SEND_ERROR "postgresql backend requested but postgresql libraries not found!") + else(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES) + message(STATUS "postgresql not enabled (postgresql libraries and/or headers not found)") + endif(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES) + endif(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR) +endif(ENABLE_POSTGRESQL OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) + # Find leveldb if(ENABLE_LEVELDB OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) find_library(LEVELDB_LIBRARY leveldb) @@ -213,9 +235,9 @@ if(ENABLE_REDIS OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) endif(REDIS_LIBRARY AND REDIS_INCLUDE_DIR) endif(ENABLE_REDIS OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES) -if(NOT USE_SQLITE3 AND NOT USE_LEVELDB AND NOT USE_REDIS) +if(NOT USE_SQLITE3 AND NOT USE_POSTGRESQL AND NOT USE_LEVELDB AND NOT USE_REDIS) message(SEND_ERROR "No database backends are configured, or none could be found") -endif(NOT USE_SQLITE3 AND NOT USE_LEVELDB AND NOT USE_REDIS) +endif(NOT USE_SQLITE3 AND NOT USE_POSTGRESQL AND NOT USE_LEVELDB AND NOT USE_REDIS) include_directories( "${PROJECT_BINARY_DIR}" @@ -247,6 +269,11 @@ if(USE_SQLITE3) set(LINK_LIBRARIES ${LINK_LIBRARIES} ${SQLITE3_LIBRARY}) endif(USE_SQLITE3) +if(USE_POSTGRESQL) + set(mapper_SRCS ${mapper_SRCS} db-postgresql.cpp) + set(LINK_LIBRARIES ${LINK_LIBRARIES} ${POSTGRESQL_LIBRARY}) +endif(USE_POSTGRESQL) + if(USE_LEVELDB) set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp) set(LINK_LIBRARIES ${LINK_LIBRARIES} ${LEVELDB_LIBRARY}) diff --git a/Changelog b/Changelog index 0a2200c..2bbfd0e 100644 --- a/Changelog +++ b/Changelog @@ -1,4 +1,7 @@ [] + Features: + - Support for postgresql backend added + Compatible with both ShadowNinja's and johnnyjoy's implementation Enhancements: - Updated the included colors file with colors for the new tiles that were recently added to the default minetest game. diff --git a/README.rst b/README.rst index 9a2fe67..f1892ab 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ Map Generation Features ----------------------- * Support for both minetest and freeminer worlds -* Support for sqlite3, leveldb and redis map databases +* Support for sqlite3, postgresql, leveldb and redis map databases * Generate a subsection of the map, or a full map (but note that the size of generated images is limited) * Generate regular maps or height-maps @@ -62,6 +62,7 @@ For more detailed instructions, see ``_. * zlib * libgd * sqlite3 (not mandatory. See ``_. +* postgresql (if postgresql support is desired) * leveldb (if leveldb support is desired) * hiredis (if redis support is desired) diff --git a/TileGenerator.cpp b/TileGenerator.cpp index a8c86f3..e7fe3b4 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -27,6 +27,9 @@ #if USE_SQLITE3 #include "db-sqlite3.h" #endif +#if USE_POSTGRESQL +#include "db-postgresql.h" +#endif #if USE_LEVELDB #include "db-leveldb.h" #endif @@ -775,6 +778,14 @@ void TileGenerator::openDb(const std::string &input) m_db = db = new DBSQLite3(input); #else unsupported = true; +#endif + } + else if (backend == "postgresql") { +#if USE_POSTGRESQL + DBPostgreSQL *db; + m_db = db = new DBPostgreSQL(input); +#else + unsupported = true; #endif } else if (backend == "leveldb") { diff --git a/cmake_config.h.in b/cmake_config.h.in index eb10906..b63efe0 100644 --- a/cmake_config.h.in +++ b/cmake_config.h.in @@ -4,6 +4,7 @@ #define CMAKE_CONFIG_H #define USE_SQLITE3 @USE_SQLITE3@ +#define USE_POSTGRESQL @USE_POSTGRESQL@ #define USE_LEVELDB @USE_LEVELDB@ #define USE_REDIS @USE_REDIS@ diff --git a/config.h b/config.h index b8f256a..dd92e34 100644 --- a/config.h +++ b/config.h @@ -36,6 +36,12 @@ #define USAGE_NAME_SQLITE #endif +#if USE_POSTGRESQL +#define USAGE_NAME_POSTGRESQL "/postgresql" +#else +#define USAGE_NAME_POSTGRESQL +#endif + #if USE_LEVELDB #define USAGE_NAME_LEVELDB "/leveldb" #else @@ -48,18 +54,20 @@ #define USAGE_NAME_REDIS #endif -#define USAGE_DATABASES "auto" USAGE_NAME_SQLITE USAGE_NAME_LEVELDB USAGE_NAME_REDIS +#define USAGE_DATABASES "auto" USAGE_NAME_SQLITE USAGE_NAME_POSTGRESQL USAGE_NAME_LEVELDB USAGE_NAME_REDIS #if !USE_SQLITE3 && !USE_LEVELDB && !USE_REDIS #error No database backends configured ! #endif // default database to use -#if USE_SQLITE3 && !USE_LEVELDB && !USE_REDIS +#if USE_SQLITE3 && !USE_POSTGRESQL && !USE_LEVELDB && !USE_REDIS #define DEFAULT_BACKEND "sqlite3" -#elif !USE_SQLITE3 && USE_LEVELDB && !USE_REDIS +#elif !USE_SQLITE3 && USE_POSTGRESQL && !USE_LEVELDB && !USE_REDIS +#define DEFAULT_BACKEND "postgresql" +#elif !USE_SQLITE3 && !USE_POSTGRESQL && USE_LEVELDB && !USE_REDIS #define DEFAULT_BACKEND "leveldb" -#elif !USE_SQLITE3 && !USE_LEVELDB && USE_REDIS +#elif !USE_SQLITE3 && !USE_POSTGRESQL && !USE_LEVELDB && USE_REDIS #define DEFAULT_BACKEND "redis" #else #define DEFAULT_BACKEND "auto" diff --git a/db-postgresql.cpp b/db-postgresql.cpp new file mode 100644 index 0000000..72a398a --- /dev/null +++ b/db-postgresql.cpp @@ -0,0 +1,127 @@ +#include "db-postgresql.h" +#include +#include // for usleep +#include +#include "Settings.h" +#include "types.h" + +#define BLOCKPOSLIST_QUERY "SELECT x, y, z FROM blocks" +#define BLOCK_QUERY "SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3" + +// From pg_type.h +#define PG_INT4OID 23 + +DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) : + m_blocksQueriedCount(0), + m_blocksReadCount(0) +{ + Settings world_mt(mapdir + "/world.mt"); + std::string connection_info; + + bool info_found = false; + // ShadowNinja's implementation + info_found = world_mt.check("postgresql_connection_info", connection_info); + // johnnyjoy's implementation + // The default value is not used here as it seems to me it has a serious issue: + // creating two worlds without specifying pg_connection_info will result in both + // worlds using the same database. + if (!info_found) + info_found = world_mt.check("pg_connection_info", connection_info); + if (!info_found) + throw std::runtime_error("Set postgresql_connection_info or pg_connection_info in world.mt to use the postgresql backend"); + + m_connection = PQconnectdb(connection_info.c_str()); + if (PQstatus(m_connection) != CONNECTION_OK) { + throw std::runtime_error(std::string("Failed to connect to postgresql database: ") + + PQerrorMessage(m_connection)); + } + + PGresult *result; + result = PQprepare(m_connection, "GetBlockPosList", BLOCKPOSLIST_QUERY, 0, NULL); + if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) + throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosList): ") + + (result ? PQresultErrorMessage(result) : "(result was NULL)")); + PQclear(result); + + result = PQprepare(m_connection, "GetBlock", BLOCK_QUERY, 0, NULL); + if (!result || PQresultStatus(result) != PGRES_COMMAND_OK) + throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlock): ") + + (result ? PQresultErrorMessage(result) : "(result was NULL)")); + PQclear(result); + + for (int i = 0; i < 3; i++) { + m_getBlockParamList[i] = reinterpret_cast(m_getBlockParams + i); + m_getBlockParamLengths[i] = sizeof(int32_t); + m_getBlockParamFormats[i] = 1; + } +} + +DBPostgreSQL::~DBPostgreSQL() +{ + PQfinish(m_connection); +} + +int DBPostgreSQL::getBlocksReadCount(void) +{ + return m_blocksReadCount; +} + +int DBPostgreSQL::getBlocksQueriedCount(void) +{ + return m_blocksQueriedCount; +} + +const DB::BlockPosList &DBPostgreSQL::getBlockPos() { + m_blockPosList.clear(); + + PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1); + if (!result || PQresultStatus(result) != PGRES_TUPLES_OK) + throw std::runtime_error(std::string("Failed to read block-pos list from database: ") + + (result ? PQresultErrorMessage(result) : "(result was NULL)")); + + int rows = PQntuples(result); + + // Make sure that we got the right data types + if (rows && + ( PQftype(result, 0) != PG_INT4OID + || PQftype(result, 1) != PG_INT4OID + || PQftype(result, 2) != PG_INT4OID)) { + throw std::runtime_error(std::string("Unexpected data type of block coordinate in database query result.")); + } + + for (int i = 0; i < rows; i++) { + int32_t x = ntohl(*reinterpret_cast(PQgetvalue(result, i, 0))); + int32_t y = ntohl(*reinterpret_cast(PQgetvalue(result, i, 1))); + int32_t z = ntohl(*reinterpret_cast(PQgetvalue(result, i, 2))); + m_blockPosList.push_back(BlockPos(x, y, z, BlockPos::XYZ)); + } + + PQclear(result); + return m_blockPosList; +} + + +DB::Block DBPostgreSQL::getBlockOnPos(const BlockPos &pos) +{ + Block block(pos,reinterpret_cast("")); + + m_blocksQueriedCount++; + + for (int i = 0; i < 3; i++) { + m_getBlockParams[i] = htonl(pos.dimension[i]); + } + + PGresult *result = PQexecPrepared(m_connection, "GetBlock", 3, m_getBlockParamList, m_getBlockParamLengths, m_getBlockParamFormats, 1); + if (!result || PQresultStatus(result) != PGRES_TUPLES_OK) + throw std::runtime_error(std::string("Failed to read block from database: ") + + (result ? PQresultErrorMessage(result) : "(result was NULL)")); + + if (PQntuples(result) != 0) { + block = Block(pos, ustring(reinterpret_cast(PQgetvalue(result, 0, 0)), PQgetlength(result, 0, 0))); + m_blocksReadCount++; + } + + PQclear(result); + return block; +} + diff --git a/db-postgresql.h b/db-postgresql.h new file mode 100644 index 0000000..79d10e0 --- /dev/null +++ b/db-postgresql.h @@ -0,0 +1,41 @@ +#ifndef _DB_POSTGRESQL_H +#define _DB_POSTGRESQL_H + +#include "db.h" +#include +#if __cplusplus >= 201103L +#include +#else +#include +#endif +#include +#include + +#include "types.h" + +class DBPostgreSQL : public DB { +#if __cplusplus >= 201103L + typedef std::unordered_map BlockCache; +#else + typedef std::map BlockCache; +#endif +public: + DBPostgreSQL(const std::string &mapdir); + virtual int getBlocksQueriedCount(void); + virtual int getBlocksReadCount(void); + virtual const BlockPosList &getBlockPos(); + virtual Block getBlockOnPos(const BlockPos &pos); + ~DBPostgreSQL(); +private: + int m_blocksQueriedCount; + int m_blocksReadCount; + PGconn *m_connection; + BlockPosList m_blockPosList; + + uint32_t m_getBlockParams[3]; + char const *m_getBlockParamList[3]; + int m_getBlockParamLengths[3]; + int m_getBlockParamFormats[3]; +}; + +#endif // _DB_POSTGRESQL_H diff --git a/doc/build-instructions.rst b/doc/build-instructions.rst index 9a06324..ca105c2 100644 --- a/doc/build-instructions.rst +++ b/doc/build-instructions.rst @@ -27,11 +27,12 @@ Libraries * zlib * libgd * sqlite3 (optional - enabled by default, set ENABLE_SQLITE3=0 in CMake to disable) +* postgresql (optional, set ENABLE_POSTGRESQL=1 in CMake to enable postgresql support) * leveldb (optional, set ENABLE_LEVELDB=1 in CMake to enable leveldb support) * hiredis (optional, set ENABLE_REDIS=1 in CMake to enable redis support) -At least one of ``sqlite3``, ``leveldb`` and ``hiredis`` is required. Check the -minetest worlds that will be mapped to know which ones should be included. +At least one of ``sqlite3``, ``postgresql``, ``leveldb`` and ``hiredis`` is required. +Check the minetest worlds that will be mapped to know which ones should be included. Build Environment ----------------- @@ -86,11 +87,11 @@ In order to make a ``.deb`` package (if desired), install the required tools: apt-get install fakeroot Finally install the minetestmapper dependencies. At least one of ``libsqlite3-dev``, -``libleveldb-dev`` and ``libhiredis-dev`` is required. +``libpq-dev``, ``libleveldb-dev`` and ``libhiredis-dev`` is required. :: - apt-get install zlib1g-dev libgd-dev libsqlite3-dev libleveldb-dev libhiredis-dev + apt-get install zlib1g-dev libgd-dev libsqlite3-dev libpq-dev libleveldb-dev libhiredis-dev Fedora and Derivatives ---------------------- @@ -120,11 +121,11 @@ In order to make an ``.rpm`` package (if desired), install the required tools: yum install rpm-build Finally install the minetestmapper dependencies. At least one of ``libsqlite3x-devel``, -``leveldb-devel`` and ``hiredis-devel`` is required. +``postgresql-devel``, ``leveldb-devel`` and ``hiredis-devel`` is required. :: - yum install zlib-devel gd-devel libsqlite3x-devel leveldb-devel hiredis-devel + yum install zlib-devel gd-devel libsqlite3x-devel postgresql-devel leveldb-devel hiredis-devel Ubuntu ------ @@ -193,6 +194,9 @@ CMake Variables ENABLE_SQLITE3: Whether to enable sqlite3 backend support (on by default) +ENABLE_POSTGRESQL: + Whether to enable postresql backend support (off by default) + ENABLE_LEVELDB: Whether to enable leveldb backend support (off by default) diff --git a/doc/features.rst b/doc/features.rst index 40146fb..da0bb5b 100644 --- a/doc/features.rst +++ b/doc/features.rst @@ -6,7 +6,7 @@ Minetestmapper generates maps of minetest and freeminer worlds. Major Features ============== * Support for both minetest and freeminer -* Support for sqlite3, leveldb and redis map databases +* Support for sqlite3, postsgresql, leveldb and redis map databases * Generate a subsection of the map, or a full map (but the size of generated images is limited - see 'Known Problems' below) diff --git a/doc/manual.rst b/doc/manual.rst index d237f50..9a44d27 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -288,7 +288,7 @@ Feedback / information options: Miscellaneous options ..................... - * ``--backend auto|sqlite3|leveldb|redis`` : Specify or override the database backend to use + * ``--backend auto|sqlite3|postgresql|leveldb|redis`` : Specify or override the database backend to use * ``--disable-blocklist-prefetch`` : Do not prefetch a block list - faster when mapping small parts of large worlds. * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` : Specify the format of the database (needed with --disable-blocklist-prefetch and a leveldb backend). @@ -313,8 +313,8 @@ Detailed Description of Options .. Contents:: :local: -``--backend auto|sqlite3|leveldb|redis`` -.......................................... +``--backend auto|sqlite3|postgresql|leveldb|redis`` +................................................... Set or override the database backend to use. By default (``auto``), the database is obtained from the world configuration, @@ -1768,7 +1768,7 @@ More information is available: .. _known problems: features.rst#known-problems -.. _--backend: `--backend auto\|sqlite3\|leveldb\|redis`_ +.. _--backend: `--backend auto\|sqlite3\|postgresql\|leveldb\|redis`_ .. _--bgcolor: `--bgcolor `_ .. _--blockcolor: `--blockcolor `_ .. _--centergeometry: `--centergeometry `_ diff --git a/package-description.txt b/package-description.txt index 8ba44b5..338ae50 100644 --- a/package-description.txt +++ b/package-description.txt @@ -6,7 +6,7 @@ map in png format. One world node is rendered as one map pixel. Features: - Standard color mapping file included (which maps a node (e.g. default:stone) to the desired color) - Easy creation of custom color mapping files (user-specific, world-specific, command-line) -- Supports all minetest backends (sqlite3, leveldb and redis) +- Supports all minetest backends (sqlite3, postgresql, leveldb and redis) - Ability to create a regular map or a height-map - Ability to draw the map at a reduced scale (up to 1:16) - Supports both minetest and freeminer worlds