From b24e6433df3c3b2926568aff9c0173459e3e8eab Mon Sep 17 00:00:00 2001 From: Ekdohibs Date: Tue, 31 Jan 2017 18:05:03 +0100 Subject: [PATCH] Add clientside translations. --- builtin/common/misc_helpers.lua | 38 +++++ doc/lua_api.txt | 66 +++++++++ src/CMakeLists.txt | 1 + src/camera.cpp | 10 +- src/chat.cpp | 1 + src/client.cpp | 16 +- src/game.cpp | 16 +- src/guiEngine.cpp | 2 +- src/guiFormSpecMenu.cpp | 31 ++-- src/guiFormSpecMenu.h | 7 +- src/hud.cpp | 6 +- src/network/clientpackethandler.cpp | 8 +- src/network/serverpackethandler.cpp | 5 +- src/server.cpp | 21 ++- src/server.h | 2 +- src/translation.cpp | 146 +++++++++++++++++++ src/translation.h | 42 ++++++ src/util/enriched_string.cpp | 6 +- src/util/string.cpp | 194 +++++++++++++++++++++++++ src/util/string.h | 56 +++++++ util/travis/clang-format-whitelist.txt | 1 + 21 files changed, 629 insertions(+), 46 deletions(-) create mode 100644 src/translation.cpp create mode 100644 src/translation.h diff --git a/builtin/common/misc_helpers.lua b/builtin/common/misc_helpers.lua index 776be29d..67eda11f 100644 --- a/builtin/common/misc_helpers.lua +++ b/builtin/common/misc_helpers.lua @@ -680,6 +680,44 @@ function core.strip_colors(str) return (str:gsub(ESCAPE_CHAR .. "%([bc]@[^)]+%)", "")) end +function core.translate(textdomain, str, ...) + local start_seq + if textdomain == "" then + start_seq = ESCAPE_CHAR .. "T" + else + start_seq = ESCAPE_CHAR .. "(T@" .. textdomain .. ")" + end + local arg = {n=select('#', ...), ...} + local end_seq = ESCAPE_CHAR .. "E" + local arg_index = 1 + local translated = str:gsub("@(.)", function(matched) + local c = string.byte(matched) + if string.byte("1") <= c and c <= string.byte("9") then + local a = c - string.byte("0") + if a ~= arg_index then + error("Escape sequences in string given to core.translate " .. + "are not in the correct order: got @" .. matched .. + "but expected @" .. tostring(arg_index)) + end + if a > arg.n then + error("Not enough arguments provided to core.translate") + end + arg_index = arg_index + 1 + return ESCAPE_CHAR .. "F" .. arg[a] .. ESCAPE_CHAR .. "E" + else + return matched + end + end) + if arg_index < arg.n + 1 then + error("Too many arguments provided to core.translate") + end + return start_seq .. translated .. end_seq +end + +function core.get_translator(textdomain) + return function(str, ...) return core.translate(textdomain or "", str, ...) end +end + -------------------------------------------------------------------------------- -- Returns the exact coordinate of a pointed surface -------------------------------------------------------------------------------- diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 32dab9d0..20c3e5a9 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -139,6 +139,7 @@ Mod directory structure | | `-- modname_something_else.png | |-- sounds | |-- media + | |-- locale | `-- `-- another @@ -182,6 +183,9 @@ Models for entities or meshnodes. Media files (textures, sounds, whatever) that will be transferred to the client and will be available for use by the mod. +### `locale` +Translation files for the clients. (See `Translations`) + Naming convention for registered textual names ---------------------------------------------- Registered names should generally be in this format: @@ -2152,6 +2156,68 @@ Helper functions * `minetest.pointed_thing_to_face_pos(placer, pointed_thing)`: returns a position * returns the exact position on the surface of a pointed node +Translations +------------ + +Texts can be translated client-side with the help of `minetest.translate` and translation files. + +### Translating a string +Two functions are provided to translate strings: `minetest.translate` and `minetest.get_translator`. + +* `minetest.get_translator(textdomain)` is a simple wrapper around `minetest.translate`, and + `minetest.get_translator(textdomain)(str, ...)` is equivalent to `minetest.translate(textdomain, str, ...)`. + It is intended to be used in the following way, so that it avoids verbose repetitions of `minetest.translate`: + + local S = minetest.get_translator(textdomain) + S(str, ...) + + As an extra commodity, if `textdomain` is nil, it is assumed to be "" instead. + +* `minetest.translate(textdomain, str, ...)` translates the string `str` with the given `textdomain` + for disambiguation. The textdomain must match the textdomain specified in the translation file in order + to get the string translated. This can be used so that a string is translated differently in different contexts. + It is advised to use the name of the mod as textdomain whenever possible, to avoid clashes with other mods. + This function must be given a number of arguments equal to the number of arguments the translated string expects. + Arguments are literal strings -- they will not be translated, so if you want them to be, they need to come as + outputs of `minetest.translate` as well. + + For instance, suppose we want to translate "@1 Wool" with "@1" being replaced by the translation of "Red". + We can do the following: + + local S = minetest.get_translator() + S("@1 Wool", S("Red")) + + This will be displayed as "Red Wool" on old clients and on clients that do not have localization enabled. + However, if we have for instance a translation file named `wool.fr.tr` containing the following: + + @1 Wool=Laine @1 + Red=Rouge + + this will be displayed as "Laine Rouge" on clients with a French locale. + +### Translation file format +A translation file has the suffix `.[lang].tr`, where `[lang]` is the language it corresponds to. +The file should be a text file, with the following format: + +* Lines beginning with `# textdomain:` (the space is significant) can be used to specify the text + domain of all following translations in the file. +* All other empty lines or lines beginning with `#` are ignored. +* Other lines should be in the format `original=translated`. Both `original` and `translated` can + contain escape sequences beginning with `@` to insert arguments, literal `@`, `=` or newline + (See ### Escapes below). There must be no extraneous whitespace around the `=` or at the beginning + or the end of the line. + +### Escapes +Strings that need to be translated can contain several escapes, preceded by `@`. +* `@@` acts as a literal `@`. +* `@n`, where `n` is a digit between 1 and 9, is an argument for the translated string that will be inlined + when translation. Due to how translations are implemented, the original translation string **must** have + its arguments in increasing order, without gaps or repetitions, starting from 1. +* `@=` acts as a literal `=`. It is not required in strings given to `minetest.translate`, but is in translation + files to avoid begin confused with the `=` separating the original from the translation. +* `@\n` (where the `\n` is a literal newline) acts as a literal newline. As with `@=`, this escape is not required + in strings given to `minetest.translate`, but is in translation files. + `minetest` namespace reference ------------------------------ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4fc6bca2..cc3d6510 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -450,6 +450,7 @@ set(common_SRCS terminal_chat_console.cpp tileanimation.cpp tool.cpp + translation.cpp treegen.cpp version.cpp voxel.cpp diff --git a/src/camera.cpp b/src/camera.cpp index 7746b65d..50e18fdd 100644 --- a/src/camera.cpp +++ b/src/camera.cpp @@ -620,10 +620,11 @@ void Camera::drawNametags() f32 transformed_pos[4] = { pos.X, pos.Y, pos.Z, 1.0f }; trans.multiplyWith1x4Matrix(transformed_pos); if (transformed_pos[3] > 0) { - std::string nametag_colorless = unescape_enriched(nametag->nametag_text); + std::wstring nametag_colorless = + unescape_translate(utf8_to_wide(nametag->nametag_text)); core::dimension2d textsize = g_fontengine->getFont()->getDimension( - utf8_to_wide(nametag_colorless).c_str()); + nametag_colorless.c_str()); f32 zDiv = transformed_pos[3] == 0.0f ? 1.0f : core::reciprocal(transformed_pos[3]); v2u32 screensize = RenderingEngine::get_video_driver()->getScreenSize(); @@ -633,8 +634,9 @@ void Camera::drawNametags() screen_pos.Y = screensize.Y * (0.5 - transformed_pos[1] * zDiv * 0.5) - textsize.Height / 2; core::rect size(0, 0, textsize.Width, textsize.Height); - g_fontengine->getFont()->draw(utf8_to_wide(nametag->nametag_text).c_str(), - size + screen_pos, nametag->nametag_color); + g_fontengine->getFont()->draw( + translate_string(utf8_to_wide(nametag->nametag_text)).c_str(), + size + screen_pos, nametag->nametag_color); } } } diff --git a/src/chat.cpp b/src/chat.cpp index 35c6329c..fd071870 100644 --- a/src/chat.cpp +++ b/src/chat.cpp @@ -650,6 +650,7 @@ ChatBackend::ChatBackend(): void ChatBackend::addMessage(std::wstring name, std::wstring text) { // Note: A message may consist of multiple lines, for example the MOTD. + text = translate_string(text); WStrfnd fnd(text); while (!fnd.at_end()) { diff --git a/src/client.cpp b/src/client.cpp index 30c1ab4d..658b1039 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -51,6 +51,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "script/scripting_client.h" #include "game.h" #include "chatmessage.h" +#include "translation.h" extern gui::IGUIEnvironment* guienv; @@ -684,8 +685,19 @@ bool Client::loadMedia(const std::string &data, const std::string &filename) return true; } - errorstream<<"Client: Don't know how to load file \"" - <loadTranslation(data); + return true; + } + + errorstream << "Client: Don't know how to load file \"" + << filename << "\"" << std::endl; return false; } diff --git a/src/game.cpp b/src/game.cpp index 68b8e076..4a26bc84 100644 --- a/src/game.cpp +++ b/src/game.cpp @@ -58,6 +58,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "sky.h" #include "subgame.h" #include "tool.h" +#include "translation.h" #include "util/basic_macros.h" #include "util/directiontables.h" #include "util/pointedthing.h" @@ -242,7 +243,7 @@ void update_profiler_gui(gui::IGUIStaticText *guitext_profiler, FontEngine *fe, std::ostringstream os(std::ios_base::binary); g_profiler->printPage(os, show_profiler, show_profiler_max); - std::wstring text = utf8_to_wide(os.str()); + std::wstring text = translate_string(utf8_to_wide(os.str())); setStaticText(guitext_profiler, text.c_str()); guitext_profiler->setVisible(true); @@ -1619,6 +1620,8 @@ bool Game::startup(bool *kill, m_invert_mouse = g_settings->getBool("invert_mouse"); m_first_loop_after_window_activation = true; + g_translations->clear(); + if (!init(map_dir, address, port, gamespec)) return false; @@ -3781,7 +3784,7 @@ void Game::handlePointingAtNode(const PointedThing &pointed, NodeMetadata *meta = map.getNodeMetadata(nodepos); if (meta) { - infotext = unescape_enriched(utf8_to_wide(meta->getString("infotext"))); + infotext = unescape_translate(utf8_to_wide(meta->getString("infotext"))); } else { MapNode n = map.getNodeNoEx(nodepos); @@ -3858,15 +3861,14 @@ void Game::handlePointingAtNode(const PointedThing &pointed, void Game::handlePointingAtObject(const PointedThing &pointed, const ItemStack &playeritem, const v3f &player_position, bool show_debug) { - infotext = unescape_enriched( + infotext = unescape_translate( utf8_to_wide(runData.selected_object->infoText())); if (show_debug) { if (!infotext.empty()) { infotext += L"\n"; } - infotext += unescape_enriched(utf8_to_wide( - runData.selected_object->debugInfoText())); + infotext += utf8_to_wide(runData.selected_object->debugInfoText()); } if (isLeftPressed()) { @@ -4399,7 +4401,7 @@ void Game::updateGui(const RunStats &stats, f32 dtime, const CameraOrientation & guitext3->setRelativePosition(rect); } - setStaticText(guitext_info, infotext.c_str()); + setStaticText(guitext_info, translate_string(infotext).c_str()); guitext_info->setVisible(flags.show_hud && g_menumgr.menuCount() == 0); float statustext_time_max = 1.5; @@ -4413,7 +4415,7 @@ void Game::updateGui(const RunStats &stats, f32 dtime, const CameraOrientation & } } - setStaticText(guitext_status, m_statustext.c_str()); + setStaticText(guitext_status, translate_string(m_statustext).c_str()); guitext_status->setVisible(!m_statustext.empty()); if (!m_statustext.empty()) { diff --git a/src/guiEngine.cpp b/src/guiEngine.cpp index 07fbd366..e9b4e54c 100644 --- a/src/guiEngine.cpp +++ b/src/guiEngine.cpp @@ -547,7 +547,7 @@ bool GUIEngine::downloadFile(const std::string &url, const std::string &target) /******************************************************************************/ void GUIEngine::setTopleftText(const std::string &text) { - m_toplefttext = utf8_to_wide(text); + m_toplefttext = translate_string(utf8_to_wide(text)); updateTopLeftTextSize(); } diff --git a/src/guiFormSpecMenu.cpp b/src/guiFormSpecMenu.cpp index 996eeed5..5ae65260 100644 --- a/src/guiFormSpecMenu.cpp +++ b/src/guiFormSpecMenu.cpp @@ -398,7 +398,7 @@ void GUIFormSpecMenu::parseCheckbox(parserData* data, const std::string &element if (selected == "true") fselected = true; - std::wstring wlabel = utf8_to_wide(unescape_string(label)); + std::wstring wlabel = translate_string(utf8_to_wide(unescape_string(label))); core::rect rect = core::rect( pos.X, pos.Y + ((imgsize.Y/2) - m_btn_height), @@ -594,7 +594,7 @@ void GUIFormSpecMenu::parseButton(parserData* data, const std::string &element, if(!data->explicit_size) warningstream<<"invalid use of button without a size[] element"<addItem(unescape_enriched(unescape_string( + e->addItem(unescape_translate(unescape_string( utf8_to_wide(item))).c_str()); } @@ -927,7 +927,7 @@ void GUIFormSpecMenu::parsePwdField(parserData* data, const std::string &element core::rect rect = core::rect(pos.X, pos.Y, pos.X+geom.X, pos.Y+geom.Y); - std::wstring wlabel = utf8_to_wide(unescape_string(label)); + std::wstring wlabel = translate_string(utf8_to_wide(unescape_string(label))); FieldSpec spec( name, @@ -999,7 +999,7 @@ void GUIFormSpecMenu::parseSimpleField(parserData* data, default_val = m_form_src->resolveText(default_val); - std::wstring wlabel = utf8_to_wide(unescape_string(label)); + std::wstring wlabel = translate_string(utf8_to_wide(unescape_string(label))); FieldSpec spec( name, @@ -1099,7 +1099,7 @@ void GUIFormSpecMenu::parseTextArea(parserData* data, std::vector& default_val = m_form_src->resolveText(default_val); - std::wstring wlabel = utf8_to_wide(unescape_string(label)); + std::wstring wlabel = translate_string(utf8_to_wide(unescape_string(label))); FieldSpec spec( name, @@ -1249,7 +1249,7 @@ void GUIFormSpecMenu::parseVertLabel(parserData* data, const std::string &elemen ((parts.size() > 2) && (m_formspec_version > FORMSPEC_API_VERSION))) { std::vector v_pos = split(parts[0],','); - std::wstring text = unescape_enriched( + std::wstring text = unescape_translate( unescape_string(utf8_to_wide(parts[1]))); MY_CHECKPOS("vertlabel",1); @@ -1436,7 +1436,7 @@ void GUIFormSpecMenu::parseTabHeader(parserData* data, const std::string &elemen e->setNotClipped(true); for (const std::string &button : buttons) { - e->addTab(unescape_enriched(unescape_string( + e->addTab(unescape_translate(unescape_string( utf8_to_wide(button))).c_str(), -1); } @@ -1495,7 +1495,7 @@ void GUIFormSpecMenu::parseItemImageButton(parserData* data, const std::string & item.deSerialize(item_name, idef); m_tooltips[name] = - TooltipSpec(item.getDefinition(idef).description, + TooltipSpec(utf8_to_wide(item.getDefinition(idef).description), m_default_tooltip_bgcolor, m_default_tooltip_color); @@ -1613,7 +1613,7 @@ void GUIFormSpecMenu::parseTooltip(parserData* data, const std::string &element) std::vector parts = split(element,';'); if (parts.size() == 2) { std::string name = parts[0]; - m_tooltips[name] = TooltipSpec(unescape_string(parts[1]), + m_tooltips[name] = TooltipSpec(utf8_to_wide(unescape_string(parts[1])), m_default_tooltip_bgcolor, m_default_tooltip_color); return; } @@ -1622,7 +1622,7 @@ void GUIFormSpecMenu::parseTooltip(parserData* data, const std::string &element) std::string name = parts[0]; video::SColor tmp_color1, tmp_color2; if ( parseColorString(parts[2], tmp_color1, false) && parseColorString(parts[3], tmp_color2, false) ) { - m_tooltips[name] = TooltipSpec(unescape_string(parts[1]), + m_tooltips[name] = TooltipSpec(utf8_to_wide(unescape_string(parts[1])), tmp_color1, tmp_color2); return; } @@ -2632,14 +2632,15 @@ void GUIFormSpecMenu::drawMenu() void GUIFormSpecMenu::showTooltip(const std::wstring &text, const irr::video::SColor &color, const irr::video::SColor &bgcolor) { + const std::wstring ntext = translate_string(text); m_tooltip_element->setOverrideColor(color); m_tooltip_element->setBackgroundColor(bgcolor); - setStaticText(m_tooltip_element, text.c_str()); + setStaticText(m_tooltip_element, ntext.c_str()); // Tooltip size and offset s32 tooltip_width = m_tooltip_element->getTextWidth() + m_btn_height; #if (IRRLICHT_VERSION_MAJOR <= 1 && IRRLICHT_VERSION_MINOR <= 8 && IRRLICHT_VERSION_REVISION < 2) || USE_FREETYPE == 1 - std::vector text_rows = str_split(text, L'\n'); + std::vector text_rows = str_split(ntext, L'\n'); s32 tooltip_height = m_tooltip_element->getTextHeight() * text_rows.size() + 5; #else s32 tooltip_height = m_tooltip_element->getTextHeight() + 5; diff --git a/src/guiFormSpecMenu.h b/src/guiFormSpecMenu.h index b22d62b1..02f367fb 100644 --- a/src/guiFormSpecMenu.h +++ b/src/guiFormSpecMenu.h @@ -203,7 +203,7 @@ class GUIFormSpecMenu : public GUIModalMenu const std::wstring &default_text, int id) : fname(name), flabel(label), - fdefault(unescape_enriched(default_text)), + fdefault(unescape_enriched(translate_string(default_text))), fid(id), send(false), ftype(f_Unknown), @@ -237,10 +237,9 @@ class GUIFormSpecMenu : public GUIModalMenu struct TooltipSpec { TooltipSpec() = default; - - TooltipSpec(const std::string &a_tooltip, irr::video::SColor a_bgcolor, + TooltipSpec(const std::wstring &a_tooltip, irr::video::SColor a_bgcolor, irr::video::SColor a_color): - tooltip(utf8_to_wide(a_tooltip)), + tooltip(translate_string(a_tooltip)), bgcolor(a_bgcolor), color(a_color) { diff --git a/src/hud.cpp b/src/hud.cpp index 339f50d4..7135bc42 100644 --- a/src/hud.cpp +++ b/src/hud.cpp @@ -317,7 +317,7 @@ void Hud::drawLuaElements(const v3s16 &camera_offset) (e->number >> 8) & 0xFF, (e->number >> 0) & 0xFF); core::rect size(0, 0, e->scale.X, text_height * e->scale.Y); - std::wstring text = unescape_enriched(utf8_to_wide(e->text)); + std::wstring text = unescape_translate(utf8_to_wide(e->text)); core::dimension2d textsize = font->getDimension(text.c_str()); v2s32 offset((e->align.X - 1.0) * (textsize.Width / 2), (e->align.Y - 1.0) * (textsize.Height / 2)); @@ -354,11 +354,11 @@ void Hud::drawLuaElements(const v3s16 &camera_offset) (e->number >> 8) & 0xFF, (e->number >> 0) & 0xFF); core::rect size(0, 0, 200, 2 * text_height); - std::wstring text = unescape_enriched(utf8_to_wide(e->name)); + std::wstring text = unescape_translate(utf8_to_wide(e->name)); font->draw(text.c_str(), size + pos, color); std::ostringstream os; os << distance << e->text; - text = unescape_enriched(utf8_to_wide(os.str())); + text = unescape_translate(utf8_to_wide(os.str())); pos.Y += text_height; font->draw(text.c_str(), size + pos, color); break; } diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index 4800ea87..f42f9219 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -36,6 +36,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/serialize.h" #include "util/srp.h" #include "tileanimation.h" +#include "gettext.h" void Client::handleCommand_Deprecated(NetworkPacket* pkt) { @@ -123,7 +124,12 @@ void Client::handleCommand_AuthAccept(NetworkPacket* pkt) << m_recommended_send_interval<getPeerId(), CSE_GotInit2); u16 protocol_version = m_clients.getProtocolVersion(pkt->getPeerId()); + std::string lang; + if (pkt->getSize() > 0) + *pkt >> lang; /* Send some initialization data @@ -632,7 +635,7 @@ void Server::handleCommand_Init2(NetworkPacket* pkt) m_clients.event(pkt->getPeerId(), CSE_SetDefinitionsSent); // Send media announcement - sendMediaAnnouncement(pkt->getPeerId()); + sendMediaAnnouncement(pkt->getPeerId(), lang); // Send detached inventories sendDetachedInventories(pkt->getPeerId()); diff --git a/src/server.cpp b/src/server.cpp index 5c09aee4..e3a26b98 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2293,6 +2293,7 @@ void Server::fillMediaCache() paths.push_back(mod.path + DIR_DELIM + "sounds"); paths.push_back(mod.path + DIR_DELIM + "media"); paths.push_back(mod.path + DIR_DELIM + "models"); + paths.push_back(mod.path + DIR_DELIM + "locale"); } paths.push_back(porting::path_user + DIR_DELIM + "textures" + DIR_DELIM + "server"); @@ -2315,6 +2316,8 @@ void Server::fillMediaCache() ".pcx", ".ppm", ".psd", ".wal", ".rgb", ".ogg", ".x", ".b3d", ".md2", ".obj", + // Custom translation file format + ".tr", NULL }; if (removeStringEnd(filename, supported_ext).empty()){ @@ -2374,7 +2377,7 @@ void Server::fillMediaCache() } } -void Server::sendMediaAnnouncement(u16 peer_id) +void Server::sendMediaAnnouncement(u16 peer_id, const std::string &lang_code) { DSTACK(FUNCTION_NAME); @@ -2382,12 +2385,22 @@ void Server::sendMediaAnnouncement(u16 peer_id) << std::endl; // Make packet - std::ostringstream os(std::ios_base::binary); - NetworkPacket pkt(TOCLIENT_ANNOUNCE_MEDIA, 0, peer_id); - pkt << (u16) m_media.size(); + + u16 media_sent = 0; + std::string lang_suffix; + lang_suffix.append(".").append(lang_code).append(".tr"); + for (const auto &i : m_media) { + if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix)) + continue; + media_sent++; + } + + pkt << media_sent; for (const auto &i : m_media) { + if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix)) + continue; pkt << i.first << i.second.sha1_digest; } diff --git a/src/server.h b/src/server.h index ab1660bf..f14a861f 100644 --- a/src/server.h +++ b/src/server.h @@ -405,7 +405,7 @@ private: void SendBlocks(float dtime); void fillMediaCache(); - void sendMediaAnnouncement(u16 peer_id); + void sendMediaAnnouncement(u16 peer_id, const std::string &lang_code); void sendRequestedMedia(u16 peer_id, const std::vector &tosend); diff --git a/src/translation.cpp b/src/translation.cpp new file mode 100644 index 00000000..e8582f32 --- /dev/null +++ b/src/translation.cpp @@ -0,0 +1,146 @@ +/* +Minetest +Copyright (C) 2017 Nore, Nathanaël Courant + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include "translation.h" +#include "log.h" +#include "util/string.h" + +static Translations main_translations; +Translations *g_translations = &main_translations; + +Translations::~Translations() +{ + clear(); +} + +void Translations::clear() +{ + m_translations.clear(); +} + +const std::wstring &Translations::getTranslation( + const std::wstring &textdomain, const std::wstring &s) +{ + std::wstring key = textdomain + L"|" + s; + try { + return m_translations.at(key); + } catch (std::out_of_range) { + warningstream << "Translations: can't find translation for string \"" + << wide_to_utf8(s) << "\" in textdomain \"" + << wide_to_utf8(textdomain) << "\"" << std::endl; + // Silence that warning in the future + m_translations[key] = s; + return s; + } +} + +void Translations::loadTranslation(const std::string &data) +{ + std::istringstream is(data); + std::wstring textdomain; + std::string line; + + while (is.good()) { + std::getline(is, line); + if (str_starts_with(line, "# textdomain:")) { + textdomain = utf8_to_wide(trim(str_split(line, ':')[1])); + } + if (line.empty() || line[0] == '#') + continue; + + std::wstring wline = utf8_to_wide(line); + if (wline.empty()) + continue; + + // Read line + // '=' marks the key-value pair, but may be escaped by an '@'. + // '\n' may also be escaped by '@'. + // All other escapes are preserved. + + size_t i = 0; + std::wostringstream word1, word2; + while (i < wline.length() && wline[i] != L'=') { + if (wline[i] == L'@') { + if (i + 1 < wline.length()) { + if (wline[i + 1] == L'=') { + word1.put(L'='); + } else { + word1.put(L'@'); + word1.put(wline[i + 1]); + } + i += 2; + } else { + // End of line, go to the next one. + word1.put(L'\n'); + if (!is.good()) { + break; + } + i = 0; + std::getline(is, line); + wline = utf8_to_wide(line); + } + } else { + word1.put(wline[i]); + i++; + } + } + + if (i == wline.length()) { + errorstream << "Malformed translation line \"" << line << "\"" + << std::endl; + continue; + } + i++; + + while (i < wline.length()) { + if (wline[i] == L'@') { + if (i + 1 < wline.length()) { + if (wline[i + 1] == L'=') { + word2.put(L'='); + } else { + word2.put(L'@'); + word2.put(wline[i + 1]); + } + i += 2; + } else { + // End of line, go to the next one. + word2.put(L'\n'); + if (!is.good()) { + break; + } + i = 0; + std::getline(is, line); + wline = utf8_to_wide(line); + } + } else { + word2.put(wline[i]); + i++; + } + } + + std::wstring oword1 = word1.str(), oword2 = word2.str(); + if (oword2.empty()) { + oword2 = oword1; + errorstream << "Ignoring empty translation for \"" + << wide_to_utf8(oword1) << "\"" << std::endl; + } + + m_translations[textdomain + L"|" + oword1] = oword2; + } +} \ No newline at end of file diff --git a/src/translation.h b/src/translation.h new file mode 100644 index 00000000..18fc6c38 --- /dev/null +++ b/src/translation.h @@ -0,0 +1,42 @@ +/* +Minetest +Copyright (C) 2017 Nore, Nathanaël Courant + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include + +class Translations; +extern Translations *g_translations; + +class Translations +{ +public: + Translations() = default; + + ~Translations(); + + void loadTranslation(const std::string &data); + void clear(); + const std::wstring &getTranslation( + const std::wstring &textdomain, const std::wstring &s); + +private: + std::unordered_map m_translations; +}; diff --git a/src/util/enriched_string.cpp b/src/util/enriched_string.cpp index f79e77d5..642188a5 100644 --- a/src/util/enriched_string.cpp +++ b/src/util/enriched_string.cpp @@ -36,19 +36,19 @@ EnrichedString::EnrichedString(const std::wstring &string, EnrichedString::EnrichedString(const std::wstring &s, const SColor &color) { clear(); - addAtEnd(s, color); + addAtEnd(translate_string(s), color); } EnrichedString::EnrichedString(const wchar_t *str, const SColor &color) { clear(); - addAtEnd(std::wstring(str), color); + addAtEnd(translate_string(std::wstring(str)), color); } void EnrichedString::operator=(const wchar_t *str) { clear(); - addAtEnd(std::wstring(str), SColor(255, 255, 255, 255)); + addAtEnd(translate_string(std::wstring(str)), SColor(255, 255, 255, 255)); } void EnrichedString::addAtEnd(const std::wstring &s, const SColor &initial_color) diff --git a/src/util/string.cpp b/src/util/string.cpp index fc2a2057..6335aeaf 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -24,6 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "hex.h" #include "../porting.h" +#include "../translation.h" #include #include @@ -743,3 +744,196 @@ void str_replace(std::string &str, char from, char to) { std::replace(str.begin(), str.end(), from, to); } + +/* Translated strings have the following format: + * \x1bT marks the beginning of a translated string + * \x1bE marks its end + * + * \x1bF marks the beginning of an argument, and \x1bE its end. + * + * Arguments are *not* translated, as they may contain escape codes. + * Thus, if you want a translated argument, it should be inside \x1bT/\x1bE tags as well. + * + * This representation is chosen so that clients ignoring escape codes will + * see untranslated strings. + * + * For instance, suppose we have a string such as "@1 Wool" with the argument "White" + * The string will be sent as "\x1bT\x1bF\x1bTWhite\x1bE\x1bE Wool\x1bE" + * To translate this string, we extract what is inside \x1bT/\x1bE tags. + * When we notice the \x1bF tag, we recursively extract what is there up to the \x1bE end tag, + * translating it as well. + * We get the argument "White", translated, and create a template string with "@1" instead of it. + * We finally get the template "@1 Wool" that was used in the beginning, which we translate + * before filling it again. + */ + +void translate_all(const std::wstring &s, size_t &i, std::wstring &res); + +void translate_string(const std::wstring &s, const std::wstring &textdomain, + size_t &i, std::wstring &res) { + std::wostringstream output; + std::vector args; + int arg_number = 1; + while (i < s.length()) { + // Not an escape sequence: just add the character. + if (s[i] != '\x1b') { + output.put(s[i]); + // The character is a literal '@'; add it twice + // so that it is not mistaken for an argument. + if (s[i] == L'@') + output.put(L'@'); + ++i; + continue; + } + + // We have an escape sequence: locate it and its data + // It is either a single character, or it begins with '(' + // and extends up to the following ')', with '\' as an escape character. + ++i; + size_t start_index = i; + size_t length; + if (i == s.length()) { + length = 0; + } else if (s[i] == L'(') { + ++i; + ++start_index; + while (i < s.length() && s[i] != L')') { + if (s[i] == L'\\') + ++i; + ++i; + } + length = i - start_index; + ++i; + if (i > s.length()) + i = s.length(); + } else { + ++i; + length = 1; + } + std::wstring escape_sequence(s, start_index, length); + + // The escape sequence is now reconstructed. + std::vector parts = split(escape_sequence, L'@'); + if (parts[0] == L"E") { + // "End of translation" escape sequence. We are done locating the string to translate. + break; + } else if (parts[0] == L"F") { + // "Start of argument" escape sequence. + // Recursively translate the argument, and add it to the argument list. + // Add an "@n" instead of the argument to the template to translate. + if (arg_number >= 10) { + errorstream << "Ignoring too many arguments to translation" << std::endl; + std::wstring arg; + translate_all(s, i, arg); + args.push_back(arg); + continue; + } + output.put(L'@'); + output << arg_number; + ++arg_number; + std::wstring arg; + translate_all(s, i, arg); + args.push_back(arg); + } else { + // This is an escape sequence *inside* the template string to translate itself. + // This should not happen, show an error message. + errorstream << "Ignoring escape sequence '" << wide_to_narrow(escape_sequence) << "' in translation" << std::endl; + } + } + + // Translate the template. + std::wstring toutput = g_translations->getTranslation(textdomain, output.str()); + + // Put back the arguments in the translated template. + std::wostringstream result; + size_t j = 0; + while (j < toutput.length()) { + // Normal character, add it to output and continue. + if (toutput[j] != L'@' || j == toutput.length() - 1) { + result.put(toutput[j]); + ++j; + continue; + } + + ++j; + // Literal escape for '@'. + if (toutput[j] == L'@') { + result.put(L'@'); + ++j; + continue; + } + + // Here we have an argument; get its index and add the translated argument to the output. + int arg_index = toutput[j] - L'1'; + ++j; + result << args[arg_index]; + } + res = result.str(); +} + +void translate_all(const std::wstring &s, size_t &i, std::wstring &res) { + std::wostringstream output; + while (i < s.length()) { + // Not an escape sequence: just add the character. + if (s[i] != '\x1b') { + output.put(s[i]); + ++i; + continue; + } + + // We have an escape sequence: locate it and its data + // It is either a single character, or it begins with '(' + // and extends up to the following ')', with '\' as an escape character. + size_t escape_start = i; + ++i; + size_t start_index = i; + size_t length; + if (i == s.length()) { + length = 0; + } else if (s[i] == L'(') { + ++i; + ++start_index; + while (i < s.length() && s[i] != L')') { + if (s[i] == L'\\') { + ++i; + } + ++i; + } + length = i - start_index; + ++i; + if (i > s.length()) + i = s.length(); + } else { + ++i; + length = 1; + } + std::wstring escape_sequence(s, start_index, length); + + // The escape sequence is now reconstructed. + std::vector parts = split(escape_sequence, L'@'); + if (parts[0] == L"E") { + // "End of argument" escape sequence. Exit. + break; + } else if (parts[0] == L"T") { + // Beginning of translated string. + std::wstring textdomain; + if (parts.size() > 1) + textdomain = parts[1]; + std::wstring translated; + translate_string(s, textdomain, i, translated); + output << translated; + } else { + // Another escape sequence, such as colors. Preserve it. + output << std::wstring(s, escape_start, i - escape_start); + } + } + + res = output.str(); +} + +std::wstring translate_string(const std::wstring &s) { + size_t i = 0; + std::wstring res; + translate_all(s, i, res); + return res; +} diff --git a/src/util/string.h b/src/util/string.h index 122262af..2840f119 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -203,6 +203,56 @@ inline bool str_starts_with(const std::basic_string &str, case_insensitive); } + +/** + * Check whether \p str ends with the string suffix. If \p case_insensitive + * is true then the check is case insensitve (default is false; i.e. case is + * significant). + * + * @param str + * @param suffix + * @param case_insensitive + * @return true if the str begins with suffix + */ +template +inline bool str_ends_with(const std::basic_string &str, + const std::basic_string &suffix, + bool case_insensitive = false) +{ + if (str.size() < suffix.size()) + return false; + + size_t start = str.size() - suffix.size(); + if (!case_insensitive) + return str.compare(start, suffix.size(), suffix) == 0; + + for (size_t i = 0; i < suffix.size(); ++i) + if (tolower(str[start + i]) != tolower(suffix[i])) + return false; + return true; +} + + +/** + * Check whether \p str ends with the string suffix. If \p case_insensitive + * is true then the check is case insensitve (default is false; i.e. case is + * significant). + * + * @param str + * @param suffix + * @param case_insensitive + * @return true if the str begins with suffix + */ +template +inline bool str_ends_with(const std::basic_string &str, + const T *suffix, + bool case_insensitive = false) +{ + return str_ends_with(str, std::basic_string(suffix), + case_insensitive); +} + + /** * Splits a string into its component parts separated by the character * \p delimiter. @@ -598,6 +648,12 @@ std::vector > split(const std::basic_string &s, T delim) return tokens; } +std::wstring translate_string(const std::wstring &s); + +inline std::wstring unescape_translate(const std::wstring &s) { + return unescape_enriched(translate_string(s)); +} + /** * Checks that all characters in \p to_check are a decimal digits. * diff --git a/util/travis/clang-format-whitelist.txt b/util/travis/clang-format-whitelist.txt index f5583565..66341d76 100644 --- a/util/travis/clang-format-whitelist.txt +++ b/util/travis/clang-format-whitelist.txt @@ -325,6 +325,7 @@ src/tileanimation.cpp src/tool.cpp src/tool.h src/touchscreengui.cpp +src/translation.cpp src/treegen.cpp src/treegen.h src/unittest/test_areastore.cpp