cd4e46ee65
Configured for a max line length of 120 characters. psf/black is very opinionated and purposely doesn't leave much room for configuration. The output is mostly OK so that should be fine for us, but some things worth noting: - Manually wrapped strings will be reflowed, so by using a line length of 120 for the sake of preserving readability for our long command calls, it also means that some manually wrapped strings are back on the same line and should be manually merged again. - Code generators using string concatenation extensively look awful, since black puts each operand on a single line. We need to refactor these generators to use more pythonic string formatting, for which many options are available (`%`, `format` or f-strings). - CI checks and a pre-commit hook will be added to ensure that future buildsystem changes are well-formatted.
287 lines
9.6 KiB
Python
287 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
from collections import OrderedDict
|
|
|
|
EXTRACT_TAGS = ["description", "brief_description", "member", "constant", "theme_item", "link"]
|
|
HEADER = """\
|
|
# LANGUAGE translation of the Godot Engine class reference.
|
|
# Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.
|
|
# Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).
|
|
# This file is distributed under the same license as the Godot source code.
|
|
#
|
|
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
|
#
|
|
#, fuzzy
|
|
msgid ""
|
|
msgstr ""
|
|
"Project-Id-Version: Godot Engine class reference\\n"
|
|
"Report-Msgid-Bugs-To: https://github.com/godotengine/godot\\n"
|
|
"MIME-Version: 1.0\\n"
|
|
"Content-Type: text/plain; charset=UTF-8\\n"
|
|
"Content-Transfer-Encoding: 8-bit\\n"
|
|
|
|
"""
|
|
# Some strings used by makerst.py are normally part of the editor translations,
|
|
# so we need to include them manually here for the online docs.
|
|
BASE_STRINGS = [
|
|
"Description",
|
|
"Tutorials",
|
|
"Properties",
|
|
"Methods",
|
|
"Theme Properties",
|
|
"Signals",
|
|
"Enumerations",
|
|
"Constants",
|
|
"Property Descriptions",
|
|
"Method Descriptions",
|
|
]
|
|
|
|
## <xml-line-number-hack from="https://stackoverflow.com/a/36430270/10846399">
|
|
import sys
|
|
|
|
sys.modules["_elementtree"] = None
|
|
import xml.etree.ElementTree as ET
|
|
|
|
## override the parser to get the line number
|
|
class LineNumberingParser(ET.XMLParser):
|
|
def _start(self, *args, **kwargs):
|
|
## Here we assume the default XML parser which is expat
|
|
## and copy its element position attributes into output Elements
|
|
element = super(self.__class__, self)._start(*args, **kwargs)
|
|
element._start_line_number = self.parser.CurrentLineNumber
|
|
element._start_column_number = self.parser.CurrentColumnNumber
|
|
element._start_byte_index = self.parser.CurrentByteIndex
|
|
return element
|
|
|
|
def _end(self, *args, **kwargs):
|
|
element = super(self.__class__, self)._end(*args, **kwargs)
|
|
element._end_line_number = self.parser.CurrentLineNumber
|
|
element._end_column_number = self.parser.CurrentColumnNumber
|
|
element._end_byte_index = self.parser.CurrentByteIndex
|
|
return element
|
|
|
|
|
|
## </xml-line-number-hack>
|
|
|
|
|
|
class Desc:
|
|
def __init__(self, line_no, msg, desc_list=None):
|
|
## line_no : the line number where the desc is
|
|
## msg : the description string
|
|
## desc_list : the DescList it belongs to
|
|
self.line_no = line_no
|
|
self.msg = msg
|
|
self.desc_list = desc_list
|
|
|
|
|
|
class DescList:
|
|
def __init__(self, doc, path):
|
|
## doc : root xml element of the document
|
|
## path : file path of the xml document
|
|
## list : list of Desc objects for this document
|
|
self.doc = doc
|
|
self.path = path
|
|
self.list = []
|
|
|
|
|
|
def print_error(error):
|
|
print("ERROR: {}".format(error))
|
|
|
|
|
|
## build classes with xml elements recursively
|
|
def _collect_classes_dir(path, classes):
|
|
if not os.path.isdir(path):
|
|
print_error("Invalid directory path: {}".format(path))
|
|
exit(1)
|
|
for _dir in map(lambda dir: os.path.join(path, dir), os.listdir(path)):
|
|
if os.path.isdir(_dir):
|
|
_collect_classes_dir(_dir, classes)
|
|
elif os.path.isfile(_dir):
|
|
if not _dir.endswith(".xml"):
|
|
# print("Got non-.xml file '{}', skipping.".format(path))
|
|
continue
|
|
_collect_classes_file(_dir, classes)
|
|
|
|
|
|
## opens a file and parse xml add to classes
|
|
def _collect_classes_file(path, classes):
|
|
if not os.path.isfile(path) or not path.endswith(".xml"):
|
|
print_error("Invalid xml file path: {}".format(path))
|
|
exit(1)
|
|
print("Collecting file: {}".format(os.path.basename(path)))
|
|
|
|
try:
|
|
tree = ET.parse(path, parser=LineNumberingParser())
|
|
except ET.ParseError as e:
|
|
print_error("Parse error reading file '{}': {}".format(path, e))
|
|
exit(1)
|
|
|
|
doc = tree.getroot()
|
|
|
|
if "name" in doc.attrib:
|
|
if "version" not in doc.attrib:
|
|
print_error("Version missing from 'doc', file: {}".format(path))
|
|
|
|
name = doc.attrib["name"]
|
|
if name in classes:
|
|
print_error("Duplicate class {} at path {}".format(name, path))
|
|
exit(1)
|
|
classes[name] = DescList(doc, path)
|
|
else:
|
|
print_error("Unknown XML file {}, skipping".format(path))
|
|
|
|
|
|
## regions are list of tuples with size 3 (start_index, end_index, indent)
|
|
## indication in string where the codeblock starts, ends, and it's indent
|
|
## if i inside the region returns the indent, else returns -1
|
|
def _get_xml_indent(i, regions):
|
|
for region in regions:
|
|
if region[0] < i < region[1]:
|
|
return region[2]
|
|
return -1
|
|
|
|
|
|
## find and build all regions of codeblock which we need later
|
|
def _make_codeblock_regions(desc, path=""):
|
|
code_block_end = False
|
|
code_block_index = 0
|
|
code_block_regions = []
|
|
while not code_block_end:
|
|
code_block_index = desc.find("[codeblock]", code_block_index)
|
|
if code_block_index < 0:
|
|
break
|
|
xml_indent = 0
|
|
while True:
|
|
## [codeblock] always have a trailing new line and some tabs
|
|
## those tabs are belongs to xml indentations not code indent
|
|
if desc[code_block_index + len("[codeblock]\n") + xml_indent] == "\t":
|
|
xml_indent += 1
|
|
else:
|
|
break
|
|
end_index = desc.find("[/codeblock]", code_block_index)
|
|
if end_index < 0:
|
|
print_error("Non terminating codeblock: {}".format(path))
|
|
exit(1)
|
|
code_block_regions.append((code_block_index, end_index, xml_indent))
|
|
code_block_index += 1
|
|
return code_block_regions
|
|
|
|
|
|
def _strip_and_split_desc(desc, code_block_regions):
|
|
desc_strip = "" ## a stripped desc msg
|
|
total_indent = 0 ## code indent = total indent - xml indent
|
|
for i in range(len(desc)):
|
|
c = desc[i]
|
|
if c == "\n":
|
|
c = "\\n"
|
|
if c == '"':
|
|
c = '\\"'
|
|
if c == "\\":
|
|
c = "\\\\" ## <element \> is invalid for msgmerge
|
|
if c == "\t":
|
|
xml_indent = _get_xml_indent(i, code_block_regions)
|
|
if xml_indent >= 0:
|
|
total_indent += 1
|
|
if xml_indent < total_indent:
|
|
c = "\\t"
|
|
else:
|
|
continue
|
|
else:
|
|
continue
|
|
desc_strip += c
|
|
if c == "\\n":
|
|
total_indent = 0
|
|
return desc_strip
|
|
|
|
|
|
## make catalog strings from xml elements
|
|
def _make_translation_catalog(classes):
|
|
unique_msgs = OrderedDict()
|
|
for class_name in classes:
|
|
desc_list = classes[class_name]
|
|
for elem in desc_list.doc.iter():
|
|
if elem.tag in EXTRACT_TAGS:
|
|
if not elem.text or len(elem.text) == 0:
|
|
continue
|
|
line_no = elem._start_line_number if elem.text[0] != "\n" else elem._start_line_number + 1
|
|
desc_str = elem.text.strip()
|
|
code_block_regions = _make_codeblock_regions(desc_str, desc_list.path)
|
|
desc_msg = _strip_and_split_desc(desc_str, code_block_regions)
|
|
desc_obj = Desc(line_no, desc_msg, desc_list)
|
|
desc_list.list.append(desc_obj)
|
|
|
|
if desc_msg not in unique_msgs:
|
|
unique_msgs[desc_msg] = [desc_obj]
|
|
else:
|
|
unique_msgs[desc_msg].append(desc_obj)
|
|
return unique_msgs
|
|
|
|
|
|
## generate the catalog file
|
|
def _generate_translation_catalog_file(unique_msgs, output):
|
|
with open(output, "w", encoding="utf8") as f:
|
|
f.write(HEADER)
|
|
for msg in BASE_STRINGS:
|
|
f.write("#: doc/tools/makerst.py\n")
|
|
f.write('msgid "{}"\n'.format(msg))
|
|
f.write('msgstr ""\n\n')
|
|
for msg in unique_msgs:
|
|
if len(msg) == 0 or msg in BASE_STRINGS:
|
|
continue
|
|
|
|
f.write("#:")
|
|
desc_list = unique_msgs[msg]
|
|
for desc in desc_list:
|
|
path = desc.desc_list.path.replace("\\", "/")
|
|
if path.startswith("./"):
|
|
path = path[2:]
|
|
f.write(" {}:{}".format(path, desc.line_no))
|
|
f.write("\n")
|
|
|
|
f.write('msgid "{}"\n'.format(msg))
|
|
f.write('msgstr ""\n\n')
|
|
|
|
## TODO: what if 'nt'?
|
|
if os.name == "posix":
|
|
print("Wrapping template at 79 characters for compatibility with Weblate.")
|
|
os.system("msgmerge -w79 {0} {0} > {0}.wrap".format(output))
|
|
shutil.move("{}.wrap".format(output), output)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--path", "-p", nargs="+", default=".", help="The directory or directories containing XML files to collect."
|
|
)
|
|
parser.add_argument("--output", "-o", default="translation_catalog.pot", help="The path to the output file.")
|
|
args = parser.parse_args()
|
|
|
|
output = os.path.abspath(args.output)
|
|
if not os.path.isdir(os.path.dirname(output)) or not output.endswith(".pot"):
|
|
print_error("Invalid output path: {}".format(output))
|
|
exit(1)
|
|
|
|
classes = OrderedDict()
|
|
for path in args.path:
|
|
if not os.path.isdir(path):
|
|
print_error("Invalid working directory path: {}".format(path))
|
|
exit(1)
|
|
|
|
print("\nCurrent working dir: {}".format(path))
|
|
|
|
path_classes = OrderedDict() ## dictionary of key=class_name, value=DescList objects
|
|
_collect_classes_dir(path, path_classes)
|
|
classes.update(path_classes)
|
|
|
|
classes = OrderedDict(sorted(classes.items(), key=lambda kv: kv[0].lower()))
|
|
unique_msgs = _make_translation_catalog(classes)
|
|
_generate_translation_catalog_file(unique_msgs, output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|