Make `--doctool` locale aware

* Adds `indent(str)` to `String`:
    * Indent the (multiline) string with the given indentation.
    * This method is added in order to keep the translated XML correctly
      indented.
* Moves the loading of tool/doc translation into
  `editor/editor_translation.{h,cpp}`.
    * This will be used from both `EditorSettings` and the doc tool from
      `main`.
* Makes use of doc translation when generating XML class references, and
  setup the translation locale based on `-l LOCALE` CLI parameter.

The XML class reference won't be translated if `-l LOCALE` parameter is
not given, or when it's `-l en`.

core/string/ustring.cpp

@@ -3507,6 +3507,27 @@ char32_t String::unicode_at(int p_idx) const {
 	return operator[](p_idx);
 }
 
+String String::indent(const String &p_prefix) const {
+	String new_string;
+	int line_start = 0;
+
+	for (int i = 0; i < length(); i++) {
+		const char32_t c = operator[](i);
+		if (c == '\n') {
+			if (i == line_start) {
+				new_string += c; // Leave empty lines empty.
+			} else {
+				new_string += p_prefix + substr(line_start, i - line_start + 1);
+			}
+			line_start = i + 1;
+		}
+	}
+	if (line_start != length()) {
+		new_string += p_prefix + substr(line_start);
+	}
+	return new_string;
+}
+
 String String::dedent() const {
 	String new_string;
 	String indent;

core/string/ustring.h

@@ -356,6 +356,7 @@ public:
 
 	String left(int p_pos) const;
 	String right(int p_pos) const;
+	String indent(const String &p_prefix) const;
 	String dedent() const;
 	String strip_edges(bool left = true, bool right = true) const;
 	String strip_escapes() const;

core/variant/variant_call.cpp

@@ -1411,6 +1411,7 @@ static void _register_variant_builtin_methods() {
 	bind_method(String, get_basename, sarray(), varray());
 	bind_method(String, plus_file, sarray("file"), varray());
 	bind_method(String, unicode_at, sarray("at"), varray());
+	bind_method(String, indent, sarray("prefix"), varray());
 	bind_method(String, dedent, sarray(), varray());
 	bind_method(String, hash, sarray(), varray());
 	bind_method(String, md5_text, sarray(), varray());

doc/classes/String.xml

@@ -124,7 +124,7 @@
 		<method name="dedent" qualifiers="const">
 			<return type="String" />
 			<description>
-				Returns a copy of the string with indentation (leading tabs and spaces) removed.
+				Returns a copy of the string with indentation (leading tabs and spaces) removed. See also [method indent] to add indentation.
 			</description>
 		</method>
 		<method name="ends_with" qualifiers="const">
@@ -243,6 +243,15 @@
 			<description>
 			</description>
 		</method>
+		<method name="indent" qualifiers="const">
+			<return type="String" />
+			<argument index="0" name="prefix" type="String" />
+			<description>
+				Returns a copy of the string with lines indented with [code]prefix[/code].
+				For example, the string can be indented with two tabs using [code]"\t\t"[/code], or four spaces using [code]"    "[/code]. The prefix can be any string so it can also be used to comment out strings with e.g. [code]"# "[/code]. See also [method dedent] to remove indentation.
+				[b]Note:[/b] Empty lines are kept empty.
+			</description>
+		</method>
 		<method name="insert" qualifiers="const">
 			<return type="String" />
 			<argument index="0" name="position" type="int" />

editor/doc_tools.cpp

@@ -37,12 +37,42 @@
 #include "core/io/dir_access.h"
 #include "core/io/marshalls.h"
 #include "core/object/script_language.h"
+#include "core/string/translation.h"
 #include "core/version.h"
 #include "scene/resources/theme.h"
 
 // Used for a hack preserving Mono properties on non-Mono builds.
 #include "modules/modules_enabled.gen.h" // For mono.
 
+static String _get_indent(const String &p_text) {
+	String indent;
+	bool has_text = false;
+	int line_start = 0;
+
+	for (int i = 0; i < p_text.length(); i++) {
+		const char32_t c = p_text[i];
+		if (c == '\n') {
+			line_start = i + 1;
+		} else if (c > 32) {
+			has_text = true;
+			indent = p_text.substr(line_start, i - line_start);
+			break; // Indentation of the first line that has text.
+		}
+	}
+	if (!has_text) {
+		return p_text;
+	}
+	return indent;
+}
+
+static String _translate_doc_string(const String &p_text) {
+	const String indent = _get_indent(p_text);
+	const String message = p_text.dedent().strip_edges();
+	const String translated = TranslationServer::get_singleton()->doc_translate(message, "");
+	// No need to restore stripped edges because they'll be stripped again later.
+	return translated.indent(indent);
+}
+
 void DocTools::merge_from(const DocTools &p_data) {
 	for (KeyValue<String, DocData::ClassDoc> &E : class_list) {
 		DocData::ClassDoc &c = E.value;
@@ -1289,7 +1319,7 @@ static void _write_method_doc(FileAccess *f, const String &p_name, Vector<DocDat
 			}
 
 			_write_string(f, 3, "<description>");
-			_write_string(f, 4, m.description.strip_edges().xml_escape());
+			_write_string(f, 4, _translate_doc_string(m.description).strip_edges().xml_escape());
 			_write_string(f, 3, "</description>");
 
 			_write_string(f, 2, "</" + p_name + ">");
@@ -1327,11 +1357,11 @@ Error DocTools::save_classes(const String &p_default_path, const Map<String, Str
 		_write_string(f, 0, header);
 
 		_write_string(f, 1, "<brief_description>");
-		_write_string(f, 2, c.brief_description.strip_edges().xml_escape());
+		_write_string(f, 2, _translate_doc_string(c.brief_description).strip_edges().xml_escape());
 		_write_string(f, 1, "</brief_description>");
 
 		_write_string(f, 1, "<description>");
-		_write_string(f, 2, c.description.strip_edges().xml_escape());
+		_write_string(f, 2, _translate_doc_string(c.description).strip_edges().xml_escape());
 		_write_string(f, 1, "</description>");
 
 		_write_string(f, 1, "<tutorials>");
@@ -1366,7 +1396,7 @@ Error DocTools::save_classes(const String &p_default_path, const Map<String, Str
 					_write_string(f, 2, "<member name=\"" + p.name + "\" type=\"" + p.type + "\" setter=\"" + p.setter + "\" getter=\"" + p.getter + "\" overrides=\"" + p.overrides + "\"" + additional_attributes + " />");
 				} else {
 					_write_string(f, 2, "<member name=\"" + p.name + "\" type=\"" + p.type + "\" setter=\"" + p.setter + "\" getter=\"" + p.getter + "\"" + additional_attributes + ">");
-					_write_string(f, 3, p.description.strip_edges().xml_escape());
+					_write_string(f, 3, _translate_doc_string(p.description).strip_edges().xml_escape());
 					_write_string(f, 2, "</member>");
 				}
 			}
@@ -1392,7 +1422,7 @@ Error DocTools::save_classes(const String &p_default_path, const Map<String, Str
 						_write_string(f, 2, "<constant name=\"" + k.name + "\" value=\"platform-dependent\">");
 					}
 				}
-				_write_string(f, 3, k.description.strip_edges().xml_escape());
+				_write_string(f, 3, _translate_doc_string(k.description).strip_edges().xml_escape());
 				_write_string(f, 2, "</constant>");
 			}
 
@@ -1412,7 +1442,7 @@ Error DocTools::save_classes(const String &p_default_path, const Map<String, Str
 					_write_string(f, 2, "<theme_item name=\"" + ti.name + "\" data_type=\"" + ti.data_type + "\" type=\"" + ti.type + "\">");
 				}
 
-				_write_string(f, 3, ti.description.strip_edges().xml_escape());
+				_write_string(f, 3, _translate_doc_string(ti.description).strip_edges().xml_escape());
 
 				_write_string(f, 2, "</theme_item>");
 			}

editor/editor_settings.cpp

@@ -33,21 +33,17 @@
 #include "core/config/project_settings.h"
 #include "core/input/input_map.h"
 #include "core/io/certs_compressed.gen.h"
-#include "core/io/compression.h"
 #include "core/io/config_file.h"
 #include "core/io/dir_access.h"
 #include "core/io/file_access.h"
-#include "core/io/file_access_memory.h"
 #include "core/io/ip.h"
 #include "core/io/resource_loader.h"
 #include "core/io/resource_saver.h"
-#include "core/io/translation_loader_po.h"
 #include "core/os/keyboard.h"
 #include "core/os/os.h"
 #include "core/version.h"
-#include "editor/doc_translations.gen.h"
 #include "editor/editor_node.h"
-#include "editor/editor_translations.gen.h"
+#include "editor/editor_translation.h"
 #include "scene/main/node.h"
 #include "scene/main/scene_tree.h"
 #include "scene/main/window.h"
@@ -369,16 +365,11 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 		}
 
 		String best;
-		EditorTranslationList *etl = _editor_translations;
+		for (const String &locale : get_editor_locales()) {
-
-		while (etl->data) {
-			const String &locale = etl->lang;
-
 			// Skip locales which we can't render properly (see above comment).
 			// Test against language code without regional variants (e.g. ur_PK).
 			String lang_code = locale.get_slice("_", 0);
 			if (locales_to_skip.find(lang_code) != -1) {
-				etl++;
 				continue;
 			}
 
@@ -392,8 +383,6 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 			if (best.is_empty() && host_lang.begins_with(locale)) {
 				best = locale;
 			}
-
-			etl++;
 		}
 
 		if (best.is_empty()) {
@@ -922,50 +911,10 @@ void EditorSettings::setup_language() {
 		return; // Default, nothing to do.
 	}
 	// Load editor translation for configured/detected locale.
-	EditorTranslationList *etl = _editor_translations;
+	load_editor_translations(lang);
-	while (etl->data) {
-		if (etl->lang == lang) {
-			Vector<uint8_t> data;
-			data.resize(etl->uncomp_size);
-			Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
-
-			FileAccessMemory *fa = memnew(FileAccessMemory);
-			fa->open_custom(data.ptr(), data.size());
-
-			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
-
-			if (tr.is_valid()) {
-				tr->set_locale(etl->lang);
-				TranslationServer::get_singleton()->set_tool_translation(tr);
-				break;
-			}
-		}
-
-		etl++;
-	}
 
 	// Load class reference translation.
-	DocTranslationList *dtl = _doc_translations;
+	load_doc_translations(lang);
-	while (dtl->data) {
-		if (dtl->lang == lang) {
-			Vector<uint8_t> data;
-			data.resize(dtl->uncomp_size);
-			Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
-
-			FileAccessMemory *fa = memnew(FileAccessMemory);
-			fa->open_custom(data.ptr(), data.size());
-
-			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
-
-			if (tr.is_valid()) {
-				tr->set_locale(dtl->lang);
-				TranslationServer::get_singleton()->set_doc_translation(tr);
-				break;
-			}
-		}
-
-		dtl++;
-	}
 }
 
 void EditorSettings::setup_network() {

editor/editor_translation.cpp

@@ -0,0 +1,99 @@
+/*************************************************************************/
+/*  editor_translation.cpp                                               */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "editor/editor_translation.h"
+
+#include "core/io/compression.h"
+#include "core/io/file_access_memory.h"
+#include "core/io/translation_loader_po.h"
+#include "editor/doc_translations.gen.h"
+#include "editor/editor_translations.gen.h"
+
+Vector<String> get_editor_locales() {
+	Vector<String> locales;
+
+	EditorTranslationList *etl = _editor_translations;
+	while (etl->data) {
+		const String &locale = etl->lang;
+		locales.push_back(locale);
+
+		etl++;
+	}
+
+	return locales;
+}
+
+void load_editor_translations(const String &p_locale) {
+	EditorTranslationList *etl = _editor_translations;
+	while (etl->data) {
+		if (etl->lang == p_locale) {
+			Vector<uint8_t> data;
+			data.resize(etl->uncomp_size);
+			Compression::decompress(data.ptrw(), etl->uncomp_size, etl->data, etl->comp_size, Compression::MODE_DEFLATE);
+
+			FileAccessMemory *fa = memnew(FileAccessMemory);
+			fa->open_custom(data.ptr(), data.size());
+
+			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
+
+			if (tr.is_valid()) {
+				tr->set_locale(etl->lang);
+				TranslationServer::get_singleton()->set_tool_translation(tr);
+				break;
+			}
+		}
+
+		etl++;
+	}
+}
+
+void load_doc_translations(const String &p_locale) {
+	DocTranslationList *dtl = _doc_translations;
+	while (dtl->data) {
+		if (dtl->lang == p_locale) {
+			Vector<uint8_t> data;
+			data.resize(dtl->uncomp_size);
+			Compression::decompress(data.ptrw(), dtl->uncomp_size, dtl->data, dtl->comp_size, Compression::MODE_DEFLATE);
+
+			FileAccessMemory *fa = memnew(FileAccessMemory);
+			fa->open_custom(data.ptr(), data.size());
+
+			Ref<Translation> tr = TranslationLoaderPO::load_translation(fa);
+
+			if (tr.is_valid()) {
+				tr->set_locale(dtl->lang);
+				TranslationServer::get_singleton()->set_doc_translation(tr);
+				break;
+			}
+		}
+
+		dtl++;
+	}
+}

editor/editor_translation.h

@@ -0,0 +1,41 @@
+/*************************************************************************/
+/*  editor_translation.h                                                 */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef EDITOR_TRANSLATION_H
+#define EDITOR_TRANSLATION_H
+
+#include "core/string/ustring.h"
+#include "core/templates/vector.h"
+
+Vector<String> get_editor_locales();
+void load_editor_translations(const String &p_locale);
+void load_doc_translations(const String &p_locale);
+
+#endif // EDITOR_TRANSLATION_H

main/main.cpp

@@ -83,6 +83,7 @@
 #include "editor/doc_tools.h"
 #include "editor/editor_node.h"
 #include "editor/editor_settings.h"
+#include "editor/editor_translation.h"
 #include "editor/progress_dialog.h"
 #include "editor/project_manager.h"
 #ifndef NO_EDITOR_SPLASH
@@ -1937,7 +1938,6 @@ Error Main::setup2(Thread::ID p_main_tid_override) {
 	}
 
 	_start_success = true;
-	locale = String();
 
 	ClassDB::set_current_api(ClassDB::API_NONE); //no more APIs are registered at this point
 
@@ -2049,6 +2049,11 @@ bool Main::start() {
 		// Needed to instance editor-only classes for their default values
 		Engine::get_singleton()->set_editor_hint(true);
 
+		// Translate the class reference only when `-l LOCALE` parameter is given.
+		if (!locale.is_empty() && locale != "en") {
+			load_doc_translations(locale);
+		}
+
 		{
 			DirAccessRef da = DirAccess::open(doc_tool_path);
 			ERR_FAIL_COND_V_MSG(!da, false, "Argument supplied to --doctool must be a valid directory path.");

tests/core/string/test_string.h

@@ -1132,6 +1132,25 @@ TEST_CASE("[String] c-escape/unescape") {
 	CHECK(s.c_escape().c_unescape() == s);
 }
 
+TEST_CASE("[String] indent") {
+	static const char *input[] = {
+		"",
+		"aaa\nbbb",
+		"\tcontains\n\tindent",
+		"empty\n\nline",
+	};
+	static const char *expected[] = {
+		"",
+		"\taaa\n\tbbb",
+		"\t\tcontains\n\t\tindent",
+		"\tempty\n\n\tline",
+	};
+
+	for (int i = 0; i < 3; i++) {
+		CHECK(String(input[i]).indent("\t") == expected[i]);
+	}
+}
+
 TEST_CASE("[String] dedent") {
 	String s = "      aaa\n    bbb";
 	String t = "aaa\nbbb";