Implement a PostgreSQL backend

master
Loic Blot 2016-05-14 11:00:42 +02:00
parent 0f184d77c8
commit ce42ff9cf7
7 changed files with 433 additions and 2 deletions

View File

@ -169,7 +169,8 @@ ENABLE_CURSES - Build with (n)curses; Enables a server side terminal (comm
ENABLE_FREETYPE - Build with FreeType2; Allows using TTF fonts ENABLE_FREETYPE - Build with FreeType2; Allows using TTF fonts
ENABLE_GETTEXT - Build with Gettext; Allows using translations ENABLE_GETTEXT - Build with Gettext; Allows using translations
ENABLE_GLES - Search for Open GLES headers & libraries and use them ENABLE_GLES - Search for Open GLES headers & libraries and use them
ENABLE_LEVELDB - Build with LevelDB; Enables use of LevelDB map backend (faster than SQLite3) ENABLE_LEVELDB - Build with LevelDB; Enables use of LevelDB map backend
ENABLE_POSTGRESQL - Build with libpq; Enables use of PostgreSQL map backend (PostgreSQL 9.5 or greater required)
ENABLE_REDIS - Build with libhiredis; Enables use of Redis map backend ENABLE_REDIS - Build with libhiredis; Enables use of Redis map backend
ENABLE_SPATIAL - Build with LibSpatial; Speeds up AreaStores ENABLE_SPATIAL - Build with LibSpatial; Speeds up AreaStores
ENABLE_SOUND - Build with OpenAL, libogg & libvorbis; in-game Sounds ENABLE_SOUND - Build with OpenAL, libogg & libvorbis; in-game Sounds
@ -203,6 +204,8 @@ IRRLICHT_LIBRARY - Path to libIrrlicht.a/libIrrlicht.so/libIrrlic
LEVELDB_INCLUDE_DIR - Only when building with LevelDB; directory that contains db.h LEVELDB_INCLUDE_DIR - Only when building with LevelDB; directory that contains db.h
LEVELDB_LIBRARY - Only when building with LevelDB; path to libleveldb.a/libleveldb.so/libleveldb.dll.a LEVELDB_LIBRARY - Only when building with LevelDB; path to libleveldb.a/libleveldb.so/libleveldb.dll.a
LEVELDB_DLL - Only when building with LevelDB on Windows; path to libleveldb.dll LEVELDB_DLL - Only when building with LevelDB on Windows; path to libleveldb.dll
POSTGRESQL_INCLUDE_DIR - Only when building with PostgreSQL; directory that contains libpq-fe.h
POSTGRESQL_LIBRARY - Only when building with PostgreSQL; path to libpq.a/libpq.so
REDIS_INCLUDE_DIR - Only when building with Redis; directory that contains hiredis.h REDIS_INCLUDE_DIR - Only when building with Redis; directory that contains hiredis.h
REDIS_LIBRARY - Only when building with Redis; path to libhiredis.a/libhiredis.so REDIS_LIBRARY - Only when building with Redis; path to libhiredis.a/libhiredis.so
SPATIAL_INCLUDE_DIR - Only when building with LibSpatial; directory that contains spatialindex/SpatialIndex.h SPATIAL_INCLUDE_DIR - Only when building with LibSpatial; directory that contains spatialindex/SpatialIndex.h

View File

@ -189,6 +189,36 @@ if(ENABLE_CURSES)
endif() endif()
endif(ENABLE_CURSES) endif(ENABLE_CURSES)
option(ENABLE_POSTGRESQL "Enable PostgreSQL backend" TRUE)
set(USE_POSTGRESQL FALSE)
if(ENABLE_POSTGRESQL)
find_program(POSTGRESQL_CONFIG_EXECUTABLE pg_config DOC "pg_config")
find_library(POSTGRESQL_LIBRARY pq)
if(POSTGRESQL_CONFIG_EXECUTABLE)
execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE} --includedir-server
OUTPUT_VARIABLE POSTGRESQL_SERVER_INCLUDE_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE}
OUTPUT_VARIABLE POSTGRESQL_CLIENT_INCLUDE_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
# This variable is case sensitive for the cmake PostgreSQL module
set(PostgreSQL_ADDITIONAL_SEARCH_PATHS ${POSTGRESQL_SERVER_INCLUDE_DIRS} ${POSTGRESQL_CLIENT_INCLUDE_DIRS})
endif()
find_package("PostgreSQL")
if(POSTGRESQL_FOUND)
set(USE_POSTGRESQL TRUE)
message(STATUS "PostgreSQL backend enabled")
# This variable is case sensitive, don't try to change it to POSTGRESQL_INCLUDE_DIR
message(STATUS "PostgreSQL includes: ${PostgreSQL_INCLUDE_DIR}")
include_directories(${PostgreSQL_INCLUDE_DIR})
else()
message(STATUS "PostgreSQL not found!")
endif()
endif(ENABLE_POSTGRESQL)
option(ENABLE_LEVELDB "Enable LevelDB backend" TRUE) option(ENABLE_LEVELDB "Enable LevelDB backend" TRUE)
set(USE_LEVELDB FALSE) set(USE_LEVELDB FALSE)
@ -361,6 +391,7 @@ set(common_SRCS
craftdef.cpp craftdef.cpp
database-dummy.cpp database-dummy.cpp
database-leveldb.cpp database-leveldb.cpp
database-postgresql.cpp
database-redis.cpp database-redis.cpp
database-sqlite3.cpp database-sqlite3.cpp
database.cpp database.cpp
@ -592,6 +623,9 @@ if(BUILD_CLIENT)
if (USE_CURSES) if (USE_CURSES)
target_link_libraries(${PROJECT_NAME} ${CURSES_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${CURSES_LIBRARIES})
endif() endif()
if (USE_POSTGRESQL)
target_link_libraries(${PROJECT_NAME} ${POSTGRESQL_LIBRARY})
endif()
if (USE_LEVELDB) if (USE_LEVELDB)
target_link_libraries(${PROJECT_NAME} ${LEVELDB_LIBRARY}) target_link_libraries(${PROJECT_NAME} ${LEVELDB_LIBRARY})
endif() endif()
@ -622,6 +656,9 @@ if(BUILD_SERVER)
if (USE_CURSES) if (USE_CURSES)
target_link_libraries(${PROJECT_NAME}server ${CURSES_LIBRARIES}) target_link_libraries(${PROJECT_NAME}server ${CURSES_LIBRARIES})
endif() endif()
if (USE_POSTGRESQL)
target_link_libraries(${PROJECT_NAME}server ${POSTGRESQL_LIBRARY})
endif()
if (USE_LEVELDB) if (USE_LEVELDB)
target_link_libraries(${PROJECT_NAME}server ${LEVELDB_LIBRARY}) target_link_libraries(${PROJECT_NAME}server ${LEVELDB_LIBRARY})
endif() endif()

View File

@ -22,6 +22,7 @@
#cmakedefine01 USE_CURSES #cmakedefine01 USE_CURSES
#cmakedefine01 USE_LEVELDB #cmakedefine01 USE_LEVELDB
#cmakedefine01 USE_LUAJIT #cmakedefine01 USE_LUAJIT
#cmakedefine01 USE_POSTGRESQL
#cmakedefine01 USE_SPATIAL #cmakedefine01 USE_SPATIAL
#cmakedefine01 USE_SYSTEM_GMP #cmakedefine01 USE_SYSTEM_GMP
#cmakedefine01 USE_REDIS #cmakedefine01 USE_REDIS

286
src/database-postgresql.cpp Normal file
View File

@ -0,0 +1,286 @@
/*
Copyright (C) 2016 Loic Blot <loic.blot@unix-experience.fr>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "config.h"
#if USE_POSTGRESQL
#include "database-postgresql.h"
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
// Without this some of the network functions are not found on mingw
#ifndef _WIN32_WINNT
#define _WIN32_WINNT 0x0501
#endif
#include <windows.h>
#include <winsock2.h>
#else
#include <netinet/in.h>
#endif
#include "log.h"
#include "exceptions.h"
#include "settings.h"
Database_PostgreSQL::Database_PostgreSQL(const Settings &conf) :
m_connect_string(""),
m_conn(NULL),
m_pgversion(0)
{
if (!conf.getNoEx("pgsql_connection", m_connect_string)) {
throw SettingNotFoundException(
"Set pgsql_connection string in world.mt to "
"use the postgresql backend\n"
"Notes:\n"
"pgsql_connection has the following form: \n"
"\tpgsql_connection = host=127.0.0.1 port=5432 user=mt_user "
"password=mt_password dbname=minetest_world\n"
"mt_user should have CREATE TABLE, INSERT, SELECT, UPDATE and "
"DELETE rights on the database.\n"
"Don't create mt_user as a SUPERUSER!");
}
connectToDatabase();
}
Database_PostgreSQL::~Database_PostgreSQL()
{
PQfinish(m_conn);
}
void Database_PostgreSQL::connectToDatabase()
{
m_conn = PQconnectdb(m_connect_string.c_str());
if (PQstatus(m_conn) != CONNECTION_OK) {
throw DatabaseException(std::string(
"PostgreSQL database error: ") +
PQerrorMessage(m_conn));
}
m_pgversion = PQserverVersion(m_conn);
/*
* We are using UPSERT feature from PostgreSQL 9.5
* to have the better performance,
* set the minimum version to 90500
*/
if (m_pgversion < 90500) {
throw DatabaseException("PostgreSQL database error: "
"Server version 9.5 or greater required.");
}
infostream << "PostgreSQL Database: Version " << m_pgversion
<< " Connection made." << std::endl;
createDatabase();
initStatements();
}
void Database_PostgreSQL::verifyDatabase()
{
if (PQstatus(m_conn) == CONNECTION_OK)
return;
PQreset(m_conn);
ping();
}
void Database_PostgreSQL::ping()
{
if (PQping(m_connect_string.c_str()) != PQPING_OK) {
throw DatabaseException(std::string(
"PostgreSQL database error: ") +
PQerrorMessage(m_conn));
}
}
bool Database_PostgreSQL::initialized() const
{
return (PQstatus(m_conn) == CONNECTION_OK);
}
void Database_PostgreSQL::initStatements()
{
prepareStatement("read_block",
"SELECT data FROM blocks "
"WHERE posX = $1::int4 AND posY = $2::int4 AND "
"posZ = $3::int4");
prepareStatement("write_block",
"INSERT INTO blocks (posX, posY, posZ, data) VALUES "
"($1::int4, $2::int4, $3::int4, $4::bytea) "
"ON CONFLICT ON CONSTRAINT blocks_pkey DO "
"UPDATE SET data = $4::bytea");
prepareStatement("delete_block", "DELETE FROM blocks WHERE "
"posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4");
prepareStatement("list_all_loadable_blocks",
"SELECT posX, posY, posZ FROM blocks");
}
PGresult *Database_PostgreSQL::checkResults(PGresult *result, bool clear)
{
ExecStatusType statusType = PQresultStatus(result);
switch (statusType) {
case PGRES_COMMAND_OK:
case PGRES_TUPLES_OK:
break;
case PGRES_FATAL_ERROR:
default:
throw DatabaseException(
std::string("PostgreSQL database error: ") +
PQresultErrorMessage(result));
}
if (clear)
PQclear(result);
return result;
}
void Database_PostgreSQL::createDatabase()
{
PGresult *result = checkResults(PQexec(m_conn,
"SELECT relname FROM pg_class WHERE relname='blocks';"),
false);
// If table doesn't exist, create it
if (!PQntuples(result)) {
static const char* dbcreate_sql = "CREATE TABLE blocks ("
"posX INT NOT NULL,"
"posY INT NOT NULL,"
"posZ INT NOT NULL,"
"data BYTEA,"
"PRIMARY KEY (posX,posY,posZ)"
");";
checkResults(PQexec(m_conn, dbcreate_sql));
}
PQclear(result);
infostream << "PostgreSQL: Game Database was inited." << std::endl;
}
void Database_PostgreSQL::beginSave()
{
verifyDatabase();
checkResults(PQexec(m_conn, "BEGIN;"));
}
void Database_PostgreSQL::endSave()
{
checkResults(PQexec(m_conn, "COMMIT;"));
}
bool Database_PostgreSQL::saveBlock(const v3s16 &pos,
const std::string &data)
{
// Verify if we don't overflow the platform integer with the mapblock size
if (data.size() > INT_MAX) {
errorstream << "Database_PostgreSQL::saveBlock: Data truncation! "
<< "data.size() over 0xFFFF (== " << data.size()
<< ")" << std::endl;
return false;
}
verifyDatabase();
s32 x, y, z;
x = htonl(pos.X);
y = htonl(pos.Y);
z = htonl(pos.Z);
const void *args[] = { &x, &y, &z, data.c_str() };
const int argLen[] = {
sizeof(x), sizeof(y), sizeof(z), (int)data.size()
};
const int argFmt[] = { 1, 1, 1, 1 };
execPrepared("write_block", ARRLEN(args), args, argLen, argFmt);
return true;
}
void Database_PostgreSQL::loadBlock(const v3s16 &pos,
std::string *block)
{
verifyDatabase();
s32 x, y, z;
x = htonl(pos.X);
y = htonl(pos.Y);
z = htonl(pos.Z);
const void *args[] = { &x, &y, &z };
const int argLen[] = { sizeof(x), sizeof(y), sizeof(z) };
const int argFmt[] = { 1, 1, 1 };
PGresult *results = execPrepared("read_block", ARRLEN(args), args,
argLen, argFmt, false);
*block = "";
if (PQntuples(results)) {
*block = std::string(PQgetvalue(results, 0, 0),
PQgetlength(results, 0, 0));
}
PQclear(results);
}
bool Database_PostgreSQL::deleteBlock(const v3s16 &pos)
{
verifyDatabase();
s32 x, y, z;
x = htonl(pos.X);
y = htonl(pos.Y);
z = htonl(pos.Z);
const void *args[] = { &x, &y, &z };
const int argLen[] = { sizeof(x), sizeof(y), sizeof(z) };
const int argFmt[] = { 1, 1, 1 };
execPrepared("read_block", ARRLEN(args), args, argLen, argFmt);
return true;
}
void Database_PostgreSQL::listAllLoadableBlocks(std::vector<v3s16> &dst)
{
verifyDatabase();
PGresult *results = execPrepared("list_all_loadable_blocks", 0,
NULL, NULL, NULL, false, false);
int numrows = PQntuples(results);
for (int row = 0; row < numrows; ++row) {
dst.push_back(pg_to_v3s16(results, 0, 0));
}
PQclear(results);
}
#endif // USE_POSTGRESQL

95
src/database-postgresql.h Normal file
View File

@ -0,0 +1,95 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef DATABASE_POSTGRESQL_HEADER
#define DATABASE_POSTGRESQL_HEADER
#include <string>
#include <libpq-fe.h>
#include "database.h"
#include "util/basic_macros.h"
class Settings;
class Database_PostgreSQL : public Database
{
public:
Database_PostgreSQL(const Settings &conf);
~Database_PostgreSQL();
void beginSave();
void endSave();
bool saveBlock(const v3s16 &pos, const std::string &data);
void loadBlock(const v3s16 &pos, std::string *block);
bool deleteBlock(const v3s16 &pos);
void listAllLoadableBlocks(std::vector<v3s16> &dst);
bool initialized() const;
private:
// Database initialization
void connectToDatabase();
void initStatements();
void createDatabase();
inline void prepareStatement(const std::string &name, const std::string &sql)
{
checkResults(PQprepare(m_conn, name.c_str(), sql.c_str(), 0, NULL));
}
// Database connectivity checks
void ping();
void verifyDatabase();
// Database usage
PGresult *checkResults(PGresult *res, bool clear = true);
inline PGresult *execPrepared(const char *stmtName, const int paramsNumber,
const void **params,
const int *paramsLengths = NULL, const int *paramsFormats = NULL,
bool clear = true, bool nobinary = true)
{
return checkResults(PQexecPrepared(m_conn, stmtName, paramsNumber,
(const char* const*) params, paramsLengths, paramsFormats,
nobinary ? 1 : 0), clear);
}
// Conversion helpers
inline int pg_to_int(PGresult *res, int row, int col)
{
return atoi(PQgetvalue(res, row, col));
}
inline v3s16 pg_to_v3s16(PGresult *res, int row, int col)
{
return v3s16(
pg_to_int(res, row, col),
pg_to_int(res, row, col + 1),
pg_to_int(res, row, col + 2)
);
}
// Attributes
std::string m_connect_string;
PGconn *m_conn;
int m_pgversion;
};
#endif

View File

@ -50,6 +50,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#if USE_REDIS #if USE_REDIS
#include "database-redis.h" #include "database-redis.h"
#endif #endif
#if USE_POSTGRESQL
#include "database-postgresql.h"
#endif
#define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")" #define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")"
@ -3240,6 +3243,10 @@ Database *ServerMap::createDatabase(
else if (name == "redis") else if (name == "redis")
return new Database_Redis(conf); return new Database_Redis(conf);
#endif #endif
#if USE_POSTGRESQL
else if (name == "postgresql")
return new Database_PostgreSQL(conf);
#endif
else else
throw BaseException(std::string("Database backend ") + name + " not supported."); throw BaseException(std::string("Database backend ") + name + " not supported.");
} }

View File

@ -17,13 +17,15 @@ if [[ $PLATFORM == "Unix" ]]; then
if [[ $TRAVIS_OS_NAME == "linux" ]]; then if [[ $TRAVIS_OS_NAME == "linux" ]]; then
sudo apt-get install libirrlicht-dev cmake libbz2-dev libpng12-dev \ sudo apt-get install libirrlicht-dev cmake libbz2-dev libpng12-dev \
libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev \ libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev \
libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev gettext libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev \
gettext libpq-dev postgresql-server-dev-all
# Linking to LevelDB is broken, use a custom build # Linking to LevelDB is broken, use a custom build
wget http://minetest.kitsunemimi.pw/libleveldb-1.18-ubuntu12.04.7z wget http://minetest.kitsunemimi.pw/libleveldb-1.18-ubuntu12.04.7z
sudo 7z x -o/usr libleveldb-1.18-ubuntu12.04.7z sudo 7z x -o/usr libleveldb-1.18-ubuntu12.04.7z
else else
brew update brew update
brew install freetype gettext hiredis irrlicht jpeg leveldb libogg libvorbis luajit brew install freetype gettext hiredis irrlicht jpeg leveldb libogg libvorbis luajit
brew upgrade postgresql
fi fi
elif [[ $PLATFORM == "Win32" ]]; then elif [[ $PLATFORM == "Win32" ]]; then
wget http://minetest.kitsunemimi.pw/mingw_w64_i686_ubuntu12.04_4.9.1.7z -O mingw.7z wget http://minetest.kitsunemimi.pw/mingw_w64_i686_ubuntu12.04_4.9.1.7z -O mingw.7z