Add postgresql support

Supports both ShadowNinja's implementation:
  https://github.com/ShadowNinja/minetest/tree/PostgreSQL
and johnnyjoy's implementation:
  https://forum.minetest.net/viewtopic.php?f=3&t=12851
  4d537fe53a

CMakeLists.txt

@@ -144,12 +144,14 @@ include(FindPackageHandleStandardArgs)
 
 # Find database(s)
 set(USE_SQLITE3 0)
+set(USE_POSTGRESQL 0)
 set(USE_LEVELDB 0)
 set(USE_REDIS 0)
 
 OPTION(ENABLE_ANY_DATABASE "Enable any available database backends" True)
 OPTION(ENABLE_ALL_DATABASES "Enable all possible database backends")
 OPTION(ENABLE_SQLITE3 "Enable sqlite3 backend" True)
+OPTION(ENABLE_POSTGRESQL "Enable postgresql backend")
 OPTION(ENABLE_LEVELDB "Enable LevelDB backend")
 OPTION(ENABLE_REDIS "Enable redis backend")
 
@@ -173,6 +175,26 @@ if(ENABLE_SQLITE3 OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
 	endif(SQLITE3_LIBRARY AND SQLITE3_INCLUDE_DIR)
 endif(ENABLE_SQLITE3 OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
 
+# Find postgresql
+if(ENABLE_POSTGRESQL OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
+	find_library(POSTGRESQL_LIBRARY pq)
+	find_path(POSTGRESQL_INCLUDE_DIR libpq-fe.h PATH_SUFFIXES postgresql)
+	message (STATUS "postgresql library: ${POSTGRESQL_LIBRARY}")
+	message (STATUS "postgresql headers: ${POSTGRESQL_INCLUDE_DIR}")
+	if(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR)
+		set(USE_POSTGRESQL 1)
+		message(STATUS "postgresql backend enabled")
+		include_directories(${POSTGRESQL_INCLUDE_DIR})
+	else(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR)
+		set(USE_POSTGRESQL 0)
+		if(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES)
+			message(SEND_ERROR "postgresql backend requested but postgresql libraries not found!")
+		else(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES)
+			message(STATUS "postgresql not enabled (postgresql libraries and/or headers not found)")
+		endif(ENABLE_POSTGRESQL OR ENABLE_ALL_DATABASES)
+	endif(POSTGRESQL_LIBRARY AND POSTGRESQL_INCLUDE_DIR)
+endif(ENABLE_POSTGRESQL OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
+
 # Find leveldb
 if(ENABLE_LEVELDB OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
 	find_library(LEVELDB_LIBRARY leveldb)
@@ -213,9 +235,9 @@ if(ENABLE_REDIS OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
 	endif(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
 endif(ENABLE_REDIS OR ENABLE_ANY_DATABASE OR ENABLE_ALL_DATABASES)
 
-if(NOT USE_SQLITE3 AND NOT USE_LEVELDB AND NOT USE_REDIS)
+if(NOT USE_SQLITE3 AND NOT USE_POSTGRESQL AND NOT USE_LEVELDB AND NOT USE_REDIS)
 	message(SEND_ERROR "No database backends are configured, or none could be found")
-endif(NOT USE_SQLITE3 AND NOT USE_LEVELDB AND NOT USE_REDIS)
+endif(NOT USE_SQLITE3 AND NOT USE_POSTGRESQL AND NOT USE_LEVELDB AND NOT USE_REDIS)
 
 include_directories(
 	"${PROJECT_BINARY_DIR}"
@@ -247,6 +269,11 @@ if(USE_SQLITE3)
 	set(LINK_LIBRARIES ${LINK_LIBRARIES} ${SQLITE3_LIBRARY})
 endif(USE_SQLITE3)
 
+if(USE_POSTGRESQL)
+	set(mapper_SRCS ${mapper_SRCS} db-postgresql.cpp)
+	set(LINK_LIBRARIES ${LINK_LIBRARIES} ${POSTGRESQL_LIBRARY})
+endif(USE_POSTGRESQL)
+
 if(USE_LEVELDB)
 	set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp)
 	set(LINK_LIBRARIES ${LINK_LIBRARIES} ${LEVELDB_LIBRARY})

Changelog

@@ -1,4 +1,7 @@
 []
+    Features:
+	- Support for postgresql backend added
+		Compatible with both ShadowNinja's and johnnyjoy's implementation
     Enhancements:
         - Updated the included colors file with colors for the new tiles that
 	 	were recently added to the default minetest game.

README.rst

@@ -22,7 +22,7 @@ Map Generation Features
 -----------------------
 
 * Support for both minetest and freeminer worlds
-* Support for sqlite3, leveldb and redis map databases
+* Support for sqlite3, postgresql, leveldb and redis map databases
 * Generate a subsection of the map, or a full map
   (but note that the size of generated images is limited)
 * Generate regular maps or height-maps
@@ -62,6 +62,7 @@ For more detailed instructions, see `<doc/build-instructions.rst>`_.
 * zlib
 * libgd
 * sqlite3 (not mandatory. See `<doc/build-instructions.rst>`_.
+* postgresql (if postgresql support is desired)
 * leveldb (if leveldb support is desired)
 * hiredis (if redis support is desired)
 

TileGenerator.cpp

@@ -27,6 +27,9 @@
 #if USE_SQLITE3
 #include "db-sqlite3.h"
 #endif
+#if USE_POSTGRESQL
+#include "db-postgresql.h"
+#endif
 #if USE_LEVELDB
 #include "db-leveldb.h"
 #endif
@@ -775,6 +778,14 @@ void TileGenerator::openDb(const std::string &input)
 		m_db = db = new DBSQLite3(input);
 #else
 		unsupported = true;
+#endif
+	}
+	else if (backend == "postgresql") {
+#if USE_POSTGRESQL
+		DBPostgreSQL *db;
+		m_db = db = new DBPostgreSQL(input);
+#else
+		unsupported = true;
 #endif
 	}
 	else if (backend == "leveldb") {

cmake_config.h.in

@@ -4,6 +4,7 @@
 #define CMAKE_CONFIG_H
 
 #define USE_SQLITE3 @USE_SQLITE3@
+#define USE_POSTGRESQL @USE_POSTGRESQL@
 #define USE_LEVELDB @USE_LEVELDB@
 #define USE_REDIS @USE_REDIS@
 

config.h

@@ -36,6 +36,12 @@
 #define USAGE_NAME_SQLITE
 #endif
 
+#if USE_POSTGRESQL
+#define USAGE_NAME_POSTGRESQL "/postgresql"
+#else
+#define USAGE_NAME_POSTGRESQL
+#endif
+
 #if USE_LEVELDB
 #define USAGE_NAME_LEVELDB "/leveldb"
 #else
@@ -48,18 +54,20 @@
 #define USAGE_NAME_REDIS
 #endif
 
-#define USAGE_DATABASES "auto" USAGE_NAME_SQLITE USAGE_NAME_LEVELDB USAGE_NAME_REDIS
+#define USAGE_DATABASES "auto" USAGE_NAME_SQLITE USAGE_NAME_POSTGRESQL USAGE_NAME_LEVELDB USAGE_NAME_REDIS
 
 #if !USE_SQLITE3 && !USE_LEVELDB && !USE_REDIS
 #error No database backends configured !
 #endif
 
 // default database to use
-#if USE_SQLITE3 && !USE_LEVELDB && !USE_REDIS
+#if USE_SQLITE3 && !USE_POSTGRESQL && !USE_LEVELDB && !USE_REDIS
 #define DEFAULT_BACKEND "sqlite3"
-#elif !USE_SQLITE3 && USE_LEVELDB && !USE_REDIS
+#elif !USE_SQLITE3 && USE_POSTGRESQL && !USE_LEVELDB && !USE_REDIS
+#define DEFAULT_BACKEND "postgresql"
+#elif !USE_SQLITE3 && !USE_POSTGRESQL && USE_LEVELDB && !USE_REDIS
 #define DEFAULT_BACKEND "leveldb"
-#elif !USE_SQLITE3 && !USE_LEVELDB && USE_REDIS
+#elif !USE_SQLITE3 && !USE_POSTGRESQL && !USE_LEVELDB && USE_REDIS
 #define DEFAULT_BACKEND "redis"
 #else
 #define DEFAULT_BACKEND "auto"

db-postgresql.cpp

@@ -0,0 +1,127 @@
+#include "db-postgresql.h"
+#include <stdexcept>
+#include <unistd.h> // for usleep
+#include <arpa/inet.h>
+#include "Settings.h"
+#include "types.h"
+
+#define BLOCKPOSLIST_QUERY	"SELECT x, y, z FROM blocks"
+#define BLOCK_QUERY		"SELECT data FROM blocks WHERE x = $1 AND y = $2 AND z = $3"
+
+// From pg_type.h
+#define PG_INT4OID		23
+
+DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) :
+	m_blocksQueriedCount(0),
+	m_blocksReadCount(0)
+{
+	Settings world_mt(mapdir + "/world.mt");
+	std::string connection_info;
+
+	bool info_found = false;
+	// ShadowNinja's implementation
+	info_found = world_mt.check("postgresql_connection_info", connection_info);
+	// johnnyjoy's implementation
+	// The default value is not used here as it seems to me it has a serious issue:
+	// creating two worlds without specifying pg_connection_info will result in both
+	// worlds using the same database.
+	if (!info_found)
+		info_found = world_mt.check("pg_connection_info", connection_info);
+	if (!info_found)
+		throw std::runtime_error("Set postgresql_connection_info or pg_connection_info in world.mt to use the postgresql backend");
+
+	m_connection = PQconnectdb(connection_info.c_str());
+	if (PQstatus(m_connection) != CONNECTION_OK) {
+		throw std::runtime_error(std::string("Failed to connect to postgresql database: ")
+			+ PQerrorMessage(m_connection));
+	}
+
+	PGresult *result;
+	result = PQprepare(m_connection, "GetBlockPosList", BLOCKPOSLIST_QUERY, 0, NULL);
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+		throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlockPosList): ")
+			+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
+	PQclear(result);
+
+	result = PQprepare(m_connection, "GetBlock", BLOCK_QUERY, 0, NULL);
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+		throw std::runtime_error(std::string("Failed to prepare PostgreSQL statement (GetBlock): ")
+			+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
+	PQclear(result);
+
+	for (int i = 0; i < 3; i++) {
+		m_getBlockParamList[i] = reinterpret_cast<char const *>(m_getBlockParams + i);
+		m_getBlockParamLengths[i] = sizeof(int32_t);
+		m_getBlockParamFormats[i] = 1;
+	}
+}
+
+DBPostgreSQL::~DBPostgreSQL()
+{
+	PQfinish(m_connection);
+}
+
+int DBPostgreSQL::getBlocksReadCount(void)
+{
+	return m_blocksReadCount;
+}
+
+int DBPostgreSQL::getBlocksQueriedCount(void)
+{
+	return m_blocksQueriedCount;
+}
+
+const DB::BlockPosList &DBPostgreSQL::getBlockPos() {
+	m_blockPosList.clear();
+
+	PGresult *result = PQexecPrepared(m_connection, "GetBlockPosList", 0, NULL, NULL, NULL, 1);
+	if (!result || PQresultStatus(result) != PGRES_TUPLES_OK)
+		throw std::runtime_error(std::string("Failed to read block-pos list from database: ")
+			+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
+
+	int rows = PQntuples(result);
+
+	// Make sure that we got the right data types
+	if (rows &&
+		(  PQftype(result, 0) != PG_INT4OID
+		|| PQftype(result, 1) != PG_INT4OID
+		|| PQftype(result, 2) != PG_INT4OID)) {
+		throw std::runtime_error(std::string("Unexpected data type of block coordinate in database query result."));
+	}
+
+	for (int i = 0; i < rows; i++) {
+		int32_t x = ntohl(*reinterpret_cast<uint32_t *>(PQgetvalue(result, i, 0)));
+		int32_t y = ntohl(*reinterpret_cast<uint32_t *>(PQgetvalue(result, i, 1)));
+		int32_t z = ntohl(*reinterpret_cast<uint32_t *>(PQgetvalue(result, i, 2)));
+		m_blockPosList.push_back(BlockPos(x, y, z, BlockPos::XYZ));
+	}
+
+	PQclear(result);
+	return m_blockPosList;
+}
+
+
+DB::Block DBPostgreSQL::getBlockOnPos(const BlockPos &pos)
+{
+	Block block(pos,reinterpret_cast<const unsigned char *>(""));
+	
+	m_blocksQueriedCount++;
+
+	for (int i = 0; i < 3; i++) {
+		m_getBlockParams[i] = htonl(pos.dimension[i]);
+	}
+
+	PGresult *result = PQexecPrepared(m_connection, "GetBlock", 3, m_getBlockParamList, m_getBlockParamLengths, m_getBlockParamFormats, 1);
+	if (!result || PQresultStatus(result) != PGRES_TUPLES_OK)
+		throw std::runtime_error(std::string("Failed to read block from database: ")
+			+ (result ? PQresultErrorMessage(result) : "(result was NULL)"));
+
+	if (PQntuples(result) != 0) {
+		block = Block(pos, ustring(reinterpret_cast<unsigned char *>(PQgetvalue(result, 0, 0)), PQgetlength(result, 0, 0)));
+		m_blocksReadCount++;
+	}
+
+	PQclear(result);
+	return block;
+}
+

db-postgresql.h

@@ -0,0 +1,41 @@
+#ifndef _DB_POSTGRESQL_H
+#define _DB_POSTGRESQL_H
+
+#include "db.h"
+#include <postgresql/libpq-fe.h>
+#if __cplusplus >= 201103L
+#include <unordered_map>
+#else
+#include <map>
+#endif
+#include <string>
+#include <sstream>
+
+#include "types.h"
+
+class DBPostgreSQL : public DB {
+#if __cplusplus >= 201103L
+	typedef std::unordered_map<int64_t, ustring>  BlockCache;
+#else
+	typedef std::map<int64_t, ustring>  BlockCache;
+#endif
+public:
+	DBPostgreSQL(const std::string &mapdir);
+	virtual int getBlocksQueriedCount(void);
+	virtual int getBlocksReadCount(void);
+	virtual const BlockPosList &getBlockPos();
+	virtual Block getBlockOnPos(const BlockPos &pos);
+	~DBPostgreSQL();
+private:
+	int m_blocksQueriedCount;
+	int m_blocksReadCount;
+	PGconn *m_connection;
+	BlockPosList m_blockPosList;
+
+	uint32_t m_getBlockParams[3];
+	char const *m_getBlockParamList[3];
+	int m_getBlockParamLengths[3];
+	int m_getBlockParamFormats[3];
+};
+
+#endif // _DB_POSTGRESQL_H

doc/build-instructions.rst

@@ -27,11 +27,12 @@ Libraries
 * zlib
 * libgd
 * sqlite3 (optional - enabled by default, set ENABLE_SQLITE3=0 in CMake to disable)
+* postgresql (optional, set ENABLE_POSTGRESQL=1 in CMake to enable postgresql support)
 * leveldb (optional, set ENABLE_LEVELDB=1 in CMake to enable leveldb support)
 * hiredis (optional, set ENABLE_REDIS=1 in CMake to enable redis support)
 
-At least one of ``sqlite3``, ``leveldb`` and ``hiredis`` is required. Check the
-minetest worlds that will be mapped to know which ones should be included.
+At least one of ``sqlite3``, ``postgresql``, ``leveldb`` and ``hiredis`` is required.
+Check the minetest worlds that will be mapped to know which ones should be included.
 
 Build Environment
 -----------------
@@ -86,11 +87,11 @@ In order to make a ``.deb`` package (if desired), install the required tools:
 	apt-get install fakeroot
 
 Finally install the minetestmapper dependencies. At least one of ``libsqlite3-dev``,
-``libleveldb-dev`` and ``libhiredis-dev`` is required.
+``libpq-dev``, ``libleveldb-dev`` and ``libhiredis-dev`` is required.
 
 ::
 
-	apt-get install zlib1g-dev libgd-dev libsqlite3-dev libleveldb-dev libhiredis-dev
+	apt-get install zlib1g-dev libgd-dev libsqlite3-dev libpq-dev libleveldb-dev libhiredis-dev
 
 Fedora and Derivatives
 ----------------------
@@ -120,11 +121,11 @@ In order to make an ``.rpm`` package (if desired), install the required tools:
 	yum install rpm-build
 
 Finally install the minetestmapper dependencies. At least one of ``libsqlite3x-devel``,
-``leveldb-devel`` and ``hiredis-devel`` is required.
+``postgresql-devel``, ``leveldb-devel`` and ``hiredis-devel`` is required.
 
 ::
 
-	yum install zlib-devel gd-devel libsqlite3x-devel leveldb-devel hiredis-devel
+	yum install zlib-devel gd-devel libsqlite3x-devel postgresql-devel leveldb-devel hiredis-devel
 
 Ubuntu
 ------
@@ -193,6 +194,9 @@ CMake Variables
 ENABLE_SQLITE3:
     Whether to enable sqlite3 backend support (on by default)
 
+ENABLE_POSTGRESQL:
+    Whether to enable postresql backend support (off by default)
+
 ENABLE_LEVELDB:
     Whether to enable leveldb backend support (off by default)
 

doc/features.rst

@@ -6,7 +6,7 @@ Minetestmapper generates maps of minetest and freeminer worlds.
 Major Features
 ==============
 * Support for both minetest and freeminer
-* Support for sqlite3, leveldb and redis map databases
+* Support for sqlite3, postsgresql, leveldb and redis map databases
 * Generate a subsection of the map, or a full map
   (but the size of generated images is limited - see
   'Known Problems' below)

doc/manual.rst

@@ -288,7 +288,7 @@ Feedback / information options:
 Miscellaneous options
 .....................
 
-    * ``--backend auto|sqlite3|leveldb|redis`` :	Specify or override the database backend to use
+    * ``--backend auto|sqlite3|postgresql|leveldb|redis`` :	Specify or override the database backend to use
     * ``--disable-blocklist-prefetch`` :		Do not prefetch a block list - faster when mapping small parts of large worlds.
     * ``--database-format minetest-i64|freeminer-axyz|mixed|query`` :	Specify the format of the database (needed with --disable-blocklist-prefetch and a leveldb backend).
 
@@ -313,8 +313,8 @@ Detailed Description of Options
 .. Contents:: :local:
 
 
-``--backend auto|sqlite3|leveldb|redis``
-..........................................
+``--backend auto|sqlite3|postgresql|leveldb|redis``
+...................................................
 	Set or override the database backend to use.
 
 	By default (``auto``), the database is obtained from the world configuration,
@@ -1768,7 +1768,7 @@ More information is available:
 
 .. _known problems: features.rst#known-problems
 
-.. _--backend: `--backend auto\|sqlite3\|leveldb\|redis`_
+.. _--backend: `--backend auto\|sqlite3\|postgresql\|leveldb\|redis`_
 .. _--bgcolor: `--bgcolor <color>`_
 .. _--blockcolor: `--blockcolor <color>`_
 .. _--centergeometry: `--centergeometry <geometry>`_

package-description.txt

@@ -6,7 +6,7 @@ map in png format. One world node is rendered as one map pixel.
 Features:
 - Standard color mapping file included (which maps a node (e.g. default:stone) to the desired color)
 - Easy creation of custom color mapping files (user-specific, world-specific, command-line)
-- Supports all minetest backends (sqlite3, leveldb and redis)
+- Supports all minetest backends (sqlite3, postgresql, leveldb and redis)
 - Ability to create a regular map or a height-map
 - Ability to draw the map at a reduced scale (up to 1:16)
 - Supports both minetest and freeminer worlds