update deprecated functions

i18n.py

@@ -4,23 +4,26 @@
 # Script to generate the template file and update the translation files.
 # Copy the script into the mod or modpack root folder and run it there.
 #
-# Copyright (C) 2019 Joachim Stolberg
+# Copyright (C) 2019 Joachim Stolberg, 2020 FaceDeer
 # LGPLv2.1+
 
 from __future__ import print_function
 import os, fnmatch, re, shutil, errno
 
+verbose = False
+
 #group 2 will be the string, groups 1 and 3 will be the delimiters (" or ')
 #See https://stackoverflow.com/questions/46967465/regex-match-text-in-either-single-or-double-quote
-#TODO: support [[]] delimiters
 pattern_lua = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL)
+pattern_lua_bracketed = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL)
 
 # Handles "concatenation" .. " of strings"
 pattern_concat = re.compile(r'["\'][\s]*\.\.[\s]*["\']', re.DOTALL)
 
 pattern_tr = re.compile(r'(.+?[^@])=(.+)')
-pattern_name = re.compile(r'^name[ ]*=[ ]*([^ ]*)')
+pattern_name = re.compile(r'^name[ ]*=[ ]*([^ \n]*)')
 pattern_tr_filename = re.compile(r'\.tr$')
+pattern_po_language_code = re.compile(r'(.*)\.po$')
 
 #attempt to read the mod's name from the mod.conf file. Returns None on failure
 def get_modname(folder):
@@ -43,6 +46,56 @@ def get_existing_tr_files(folder):
                 out.append(name)
     return out
 
+# A series of search and replaces that massage a .po file's contents into
+# a .tr file's equivalent
+def process_po_file(text):
+    # The first three items are for unused matches
+    text = re.sub(r'#~ msgid "', "", text)
+    text = re.sub(r'"\n#~ msgstr ""\n"', "=", text)
+    text = re.sub(r'"\n#~ msgstr "', "=", text)
+    # comment lines
+    text = re.sub(r'#.*\n', "", text)
+    # converting msg pairs into "=" pairs
+    text = re.sub(r'msgid "', "", text)
+    text = re.sub(r'"\nmsgstr ""\n"', "=", text)
+    text = re.sub(r'"\nmsgstr "', "=", text)
+    # various line breaks and escape codes
+    text = re.sub(r'"\n"', "", text)
+    text = re.sub(r'"\n', "\n", text)
+    text = re.sub(r'\\"', '"', text)
+    text = re.sub(r'\\n', '@n', text)
+    # remove header text
+    text = re.sub(r'=Project-Id-Version:.*\n', "", text)
+    # remove double-spaced lines
+    text = re.sub(r'\n\n', '\n', text)
+    return text
+
+# Go through existing .po files and, if a .tr file for that language
+# *doesn't* exist, convert it and create it.
+# The .tr file that results will subsequently be reprocessed so
+# any "no longer used" strings will be preserved.
+# Note that "fuzzy" tags will be lost in this process.
+def process_po_files(folder, modname):
+    for root, dirs, files in os.walk(folder + 'locale/'):
+        for name in files:
+            code_match = pattern_po_language_code.match(name)
+            if code_match == None:
+                continue
+            language_code = code_match.group(1)
+            tr_name = modname + "." + language_code + ".tr"
+            tr_file = os.path.join(root, tr_name)
+            if os.path.exists(tr_file):
+                if verbose:
+                    print(tr_name + " already exists, ignoring " + name)
+                continue
+            fname = os.path.join(root, name)
+            with open(fname, "r", encoding='utf-8') as po_file:
+                if verbose:
+                    print("Importing translations from " + name)
+                text = process_po_file(po_file.read())
+                with open(tr_file, "wt", encoding='utf-8') as tr_out:
+                    tr_out.write(text)
+
 # from https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612
 # Creates a directory if it doesn't exist, silently does
 # nothing if it already exists
@@ -54,24 +107,68 @@ def mkdir_p(path):
             pass
         else: raise
 
+# Converts the template dictionary to a text to be written as a file
+# dKeyStrings is a dictionary of localized string to source file sets
+# dOld is a dictionary of existing translations, for use when updating
+# existing .tr files
+def strings_to_text(dkeyStrings, dOld, mod_name):
+    lOut = ["# textdomain: %s\n" % mod_name]
+    
+    dGroupedBySource = {}
+
+    for key in dkeyStrings:
+        sourceList = list(dkeyStrings[key])
+        sourceList.sort()
+        sourceString = "\n".join(sourceList)
+        listForSource = dGroupedBySource.get(sourceString, [])
+        listForSource.append(key)
+        dGroupedBySource[sourceString] = listForSource
+    
+    lSourceKeys = list(dGroupedBySource.keys())
+    lSourceKeys.sort()
+    for source in lSourceKeys:
+        lOut.append("")
+        localizedStrings = dGroupedBySource[source]
+        localizedStrings.sort()
+        lOut.append(source)
+        for localizedString in localizedStrings:
+            val = dOld.get(localizedString, "")
+            lOut.append("%s=%s" % (localizedString, val))
+
+    unusedExist = False
+    for key in dOld:
+        if key not in dkeyStrings:
+            if not unusedExist:
+                unusedExist = True
+                lOut.append("\n##### not used anymore #####")
+            lOut.append("%s=%s" % (key, dOld[key]))
+    return "\n".join(lOut)
+
 # Writes a template.txt file
-def write_template(templ_file, lkeyStrings):
+# dkeyStrings is the dictionary returned by generate_template
-    lOut = []
+def write_template(templ_file, dkeyStrings, mod_name):
-    lkeyStrings.sort()
+    text = strings_to_text(dkeyStrings, {}, mod_name)
-    for s in lkeyStrings:
-        lOut.append("%s=" % s)
     mkdir_p(os.path.dirname(templ_file))
     with open(templ_file, "wt", encoding='utf-8') as template_file:
-        template_file.write("\n".join(lOut))
+        template_file.write(text)
+
 
 # Gets all translatable strings from a lua file
 def read_lua_file_strings(lua_file):
     lOut = []
     with open(lua_file, encoding='utf-8') as text_file:
         text = text_file.read()
+        #TODO remove comments here
+
         text = re.sub(pattern_concat, "", text)
+
+        strings = []
         for s in pattern_lua.findall(text):
-            s = s[1]
+            strings.append(s[1])
+        for s in pattern_lua_bracketed.findall(text):
+            strings.append(s)
+                
+        for s in strings:
             s = re.sub(r'"\.\.\s+"', "", s)
             s = re.sub("@[^@=0-9]", "@@", s)
             s = s.replace('\\"', '"')
@@ -83,10 +180,16 @@ def read_lua_file_strings(lua_file):
     return lOut
 
 # Gets strings from an existing translation file
+# returns both a dictionary of translations
+# and the full original source text so that the new text
+# can be compared to it for changes.
 def import_tr_file(tr_file):
     dOut = {}
+    text = None
     if os.path.exists(tr_file):
         with open(tr_file, "r", encoding='utf-8') as existing_file :
+            text = existing_file.read()
+            existing_file.seek(0)
             for line in existing_file.readlines():
                 s = line.strip()
                 if s == "" or s[0] == "#":
@@ -94,53 +197,61 @@ def import_tr_file(tr_file):
                 match = pattern_tr.match(s)
                 if match:
                     dOut[match.group(1)] = match.group(2)
-    return dOut
+    return (dOut, text)
 
 # Walks all lua files in the mod folder, collects translatable strings,
 # and writes it to a template.txt file
-def generate_template(folder):
+# Returns a dictionary of localized strings to source file sets
-    lOut = []
+# that can be used with the strings_to_text function.
+def generate_template(folder, mod_name):
+    dOut = {}
     for root, dirs, files in os.walk(folder):
         for name in files:
             if fnmatch.fnmatch(name, "*.lua"):
                 fname = os.path.join(root, name)
                 found = read_lua_file_strings(fname)
+                if verbose:
                     print(fname + ": " + str(len(found)) + " translatable strings")
-                lOut.extend(found)
+
-    lOut = list(set(lOut))
+                for s in found:
-    lOut.sort()
+                    sources = dOut.get(s, set())
-    if len(lOut) == 0:
+                    sources.add("# " + fname)
+                    dOut[s] = sources
+                    
+    if len(dOut) == 0:
         return None
     templ_file = folder + "locale/template.txt"
-    write_template(templ_file, lOut)
+    write_template(templ_file, dOut, mod_name)
-    return lOut
+    return dOut
 
 # Updates an existing .tr file, copying the old one to a ".old" file
-def update_tr_file(lNew, mod_name, tr_file):
+# if any changes have happened
+# dNew is the data used to generate the template, it has all the
+# currently-existing localized strings
+def update_tr_file(dNew, mod_name, tr_file):
+    if verbose:
         print("updating " + tr_file)
-    lOut = ["# textdomain: %s\n" % mod_name]
 
-    #TODO only make a .old if there are actual changes from the old file
+    tr_import = import_tr_file(tr_file)
-    if os.path.exists(tr_file):
+    dOld = tr_import[0]
+    textOld = tr_import[1]
+
+    textNew = strings_to_text(dNew, dOld, mod_name)
+
+    if textOld and textOld != textNew:
+        print(tr_file + " has changed.")
         shutil.copyfile(tr_file, tr_file+".old")
 
-    dOld = import_tr_file(tr_file)
-    for key in lNew:
-        val = dOld.get(key, "")
-        lOut.append("%s=%s" % (key, val))
-    lOut.append("##### not used anymore #####")
-    for key in dOld:
-        if key not in lNew:
-            lOut.append("%s=%s" % (key, dOld[key]))
     with open(tr_file, "w", encoding='utf-8') as new_tr_file:
-        new_tr_file.write("\n".join(lOut))
+        new_tr_file.write(textNew)
 
 # Updates translation files for the mod in the given folder
 def update_mod(folder):
     modname = get_modname(folder)
     if modname is not None:
+        process_po_files(folder, modname)
         print("Updating translations for " + modname)
-        data = generate_template(folder)
+        data = generate_template(folder, modname)
         if data == None:
             print("No translatable strings found in " + modname)
         else:
@@ -149,6 +260,8 @@ def update_mod(folder):
     else:
         print("Unable to find modname in folder " + folder)
 
+# Determines if the folder being pointed to is a mod or a mod pack
+# and then runs update_mod accordingly
 def update_folder(folder):
     is_modpack = os.path.exists(folder+"modpack.txt") or os.path.exists(folder+"modpack.conf")
     if is_modpack:

init.lua

@@ -11,18 +11,14 @@
 
 local S = minetest.get_translator(minetest.get_current_modname())
 
-local glow = minetest.setting_get("breadcrumbs_glow_in_the_dark")
 local glow_level
-if (glow == "true") or (glow == nil) then
+if minetest.settings:get_bool("breadcrumbs_glow_in_the_dark", true) then
 	glow_level = 4
 else
 	glow_level = 0
 end
 
-local particles = minetest.setting_getbool("breadcrumbs_particles")
+local particles = minetest.settings:get_bool("breadcrumbs_particles", true)
-if particles == nil then
-	particles = true -- default true
-end
 
 local gui_bg, gui_bg_img, wood_sounds
 if minetest.get_modpath("default") then

locale/template.txt

@@ -1,3 +1,5 @@
+
+# ../breadcrumbs/init.lua
 @1 #@2@nPlaced by @3=
 @1m from last marker=
 A blank path marker sign, ready to have a label affixed=