From 3d2dd79ecd2c8456ba9401f6b12333d01f61e13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Wed, 25 Mar 2020 14:36:03 +0100 Subject: [PATCH] SCons: Drop support for Python 2 We now require SCons 3.0+ (first version with Python 3 support), and we set min required Python 3 version to 3.5 (3.4 and earlier are EOL). --- SConstruct | 5 +- compat.py | 68 ------------------- core/core_builders.py | 46 +++++++++---- doc/Makefile | 7 -- editor/SCsub | 5 +- editor/editor_builders.py | 17 +++-- editor/icons/editor_icons_builders.py | 3 +- main/main_builders.py | 7 +- methods.py | 11 ++- modules/freetype/SCsub | 4 +- .../build_scripts/make_android_mono_config.py | 3 +- modules/mono/build_scripts/mono_reg_utils.py | 9 +-- platform/SCsub | 4 +- platform_methods.py | 5 +- 14 files changed, 61 insertions(+), 133 deletions(-) delete mode 100644 compat.py diff --git a/SConstruct b/SConstruct index 4ebc33f593e..84400e800dc 100644 --- a/SConstruct +++ b/SConstruct @@ -1,6 +1,7 @@ #!/usr/bin/env python -EnsureSConsVersion(0, 98, 1) +EnsureSConsVersion(3, 0, 0) +EnsurePythonVersion(3, 5) # System import glob @@ -13,7 +14,7 @@ import methods import gles_builders from platform_methods import run_in_subprocess -# scan possible build platforms +# Scan possible build platforms platform_list = [] # list of platforms platform_opts = {} # options for each platform diff --git a/compat.py b/compat.py deleted file mode 100644 index de99eef9c28..00000000000 --- a/compat.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys - -if sys.version_info < (3,): - def isbasestring(s): - return isinstance(s, basestring) - def open_utf8(filename, mode): - return open(filename, mode) - def byte_to_str(x): - return str(ord(x)) - import cStringIO - def StringIO(): - return cStringIO.StringIO() - def encode_utf8(x): - return x - def decode_utf8(x): - return x - def iteritems(d): - return d.iteritems() - def itervalues(d): - return d.itervalues() - def escape_string(s): - if isinstance(s, unicode): - s = s.encode('ascii') - result = '' - for c in s: - if not (32 <= ord(c) < 127) or c in ('\\', '"'): - result += '\\%03o' % ord(c) - else: - result += c - return result - -else: - def isbasestring(s): - return isinstance(s, (str, bytes)) - def open_utf8(filename, mode): - return open(filename, mode, encoding="utf-8") - def byte_to_str(x): - return str(x) - import io - def StringIO(): - return io.StringIO() - import codecs - def encode_utf8(x): - return codecs.utf_8_encode(x)[0] - def decode_utf8(x): - return codecs.utf_8_decode(x)[0] - def iteritems(d): - return iter(d.items()) - def itervalues(d): - return iter(d.values()) - def charcode_to_c_escapes(c): - rev_result = [] - while c >= 256: - c, low = (c // 256, c % 256) - rev_result.append('\\%03o' % low) - rev_result.append('\\%03o' % c) - return ''.join(reversed(rev_result)) - def escape_string(s): - result = '' - if isinstance(s, str): - s = s.encode('utf-8') - for c in s: - if not(32 <= c < 127) or c in (ord('\\'), ord('"')): - result += charcode_to_c_escapes(c) - else: - result += chr(c) - return result - diff --git a/core/core_builders.py b/core/core_builders.py index 571e542c7f4..a06b61cb9b2 100644 --- a/core/core_builders.py +++ b/core/core_builders.py @@ -4,15 +4,33 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki """ from platform_methods import subprocess_main -from compat import iteritems, itervalues, open_utf8, escape_string, byte_to_str + + +def escape_string(s): + def charcode_to_c_escapes(c): + rev_result = [] + while c >= 256: + c, low = (c // 256, c % 256) + rev_result.append('\\%03o' % low) + rev_result.append('\\%03o' % c) + return ''.join(reversed(rev_result)) + + result = '' + if isinstance(s, str): + s = s.encode('utf-8') + for c in s: + if not(32 <= c < 127) or c in (ord('\\'), ord('"')): + result += charcode_to_c_escapes(c) + else: + result += chr(c) + return result def make_certs_header(target, source, env): - src = source[0] dst = target[0] f = open(src, "rb") - g = open_utf8(dst, "w") + g = open(dst, "w", encoding="utf-8") buf = f.read() decomp_size = len(buf) import zlib @@ -32,7 +50,7 @@ def make_certs_header(target, source, env): g.write("static const int _certs_uncompressed_size = " + str(decomp_size) + ";\n") g.write("static const unsigned char _certs_compressed[] = {\n") for i in range(len(buf)): - g.write("\t" + byte_to_str(buf[i]) + ",\n") + g.write("\t" + str(buf[i]) + ",\n") g.write("};\n") g.write("#endif // CERTS_COMPRESSED_GEN_H") @@ -46,8 +64,8 @@ def make_authors_header(target, source, env): src = source[0] dst = target[0] - f = open_utf8(src, "r") - g = open_utf8(dst, "w") + f = open(src, "r", encoding="utf-8") + g = open(dst, "w", encoding="utf-8") g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") g.write("#ifndef AUTHORS_GEN_H\n") @@ -92,8 +110,8 @@ def make_donors_header(target, source, env): src = source[0] dst = target[0] - f = open_utf8(src, "r") - g = open_utf8(dst, "w") + f = open(src, "r", encoding="utf-8") + g = open(dst, "w", encoding="utf-8") g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") g.write("#ifndef DONORS_GEN_H\n") @@ -163,7 +181,7 @@ def make_license_header(target, source, env): projects = OrderedDict() license_list = [] - with open_utf8(src_copyright, "r") as copyright_file: + with open(src_copyright, "r", encoding="utf-8") as copyright_file: reader = LicenseReader(copyright_file) part = {} while reader.current: @@ -183,21 +201,21 @@ def make_license_header(target, source, env): reader.next_line() data_list = [] - for project in itervalues(projects): + for project in iter(projects.values()): for part in project: part["file_index"] = len(data_list) data_list += part["Files"] part["copyright_index"] = len(data_list) data_list += part["Copyright"] - with open_utf8(dst, "w") as f: + with open(dst, "w", encoding="utf-8") as f: f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") f.write("#ifndef LICENSE_GEN_H\n") f.write("#define LICENSE_GEN_H\n") f.write("const char *const GODOT_LICENSE_TEXT =") - with open_utf8(src_license, "r") as license_file: + with open(src_license, "r", encoding="utf-8") as license_file: for line in license_file: escaped_string = escape_string(line.strip()) f.write("\n\t\t\"" + escaped_string + "\\n\"") @@ -225,7 +243,7 @@ def make_license_header(target, source, env): f.write("const ComponentCopyrightPart COPYRIGHT_PROJECT_PARTS[] = {\n") part_index = 0 part_indexes = {} - for project_name, project in iteritems(projects): + for project_name, project in iter(projects.items()): part_indexes[project_name] = part_index for part in project: f.write("\t{ \"" + escape_string(part["License"][0]) + "\", " @@ -239,7 +257,7 @@ def make_license_header(target, source, env): f.write("const int COPYRIGHT_INFO_COUNT = " + str(len(projects)) + ";\n") f.write("const ComponentCopyright COPYRIGHT_INFO[] = {\n") - for project_name, project in iteritems(projects): + for project_name, project in iter(projects.items()): f.write("\t{ \"" + escape_string(project_name) + "\", " + "©RIGHT_PROJECT_PARTS[" + str(part_indexes[project_name]) + "], " + str(len(project)) + " },\n") diff --git a/doc/Makefile b/doc/Makefile index 7f3f7ea939a..9534da9bd53 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -13,13 +13,6 @@ doxygen: mkdir -p $(OUTPUTDIR)/doxygen doxygen Doxyfile -markdown: - rm -rf $(OUTPUTDIR)/markdown - mkdir -p $(OUTPUTDIR)/markdown - pushd $(OUTPUTDIR)/markdown - python2 $(TOOLSDIR)/makemd.py $(CLASSES) - popd - rst: rm -rf $(OUTPUTDIR)/rst mkdir -p $(OUTPUTDIR)/rst diff --git a/editor/SCsub b/editor/SCsub index 6e8679b770f..61562d70d36 100644 --- a/editor/SCsub +++ b/editor/SCsub @@ -7,13 +7,12 @@ env.editor_sources = [] import os import os.path from platform_methods import run_in_subprocess -from compat import open_utf8 import editor_builders def _make_doc_data_class_path(to_path): # NOTE: It is safe to generate this file here, since this is still executed serially - g = open_utf8(os.path.join(to_path, "doc_data_class_path.gen.h"), "w") + g = open(os.path.join(to_path, "doc_data_class_path.gen.h"), "w", encoding="utf-8") g.write("static const int _doc_data_class_path_count = " + str(len(env.doc_class_path)) + ";\n") g.write("struct _DocDataClassPath { const char* name; const char* path; };\n") @@ -37,7 +36,7 @@ if env['tools']: reg_exporters += '}\n' # NOTE: It is safe to generate this file here, since this is still executed serially - with open_utf8("register_exporters.gen.cpp", "w") as f: + with open("register_exporters.gen.cpp", "w", encoding="utf-8") as f: f.write(reg_exporters_inc) f.write(reg_exporters) diff --git a/editor/editor_builders.py b/editor/editor_builders.py index e8c23acf9e8..44c3e50dfc8 100644 --- a/editor/editor_builders.py +++ b/editor/editor_builders.py @@ -6,24 +6,23 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki import os import os.path from platform_methods import subprocess_main -from compat import encode_utf8, byte_to_str, open_utf8 def make_doc_header(target, source, env): dst = target[0] - g = open_utf8(dst, "w") + g = open(dst, "w", encoding="utf-8") buf = "" docbegin = "" docend = "" for src in source: if not src.endswith(".xml"): continue - with open_utf8(src, "r") as f: + with open(src, "r", encoding="utf-8") as f: content = f.read() buf += content - buf = encode_utf8(docbegin + buf + docend) + buf = (docbegin + buf + docend).encode("utf-8") decomp_size = len(buf) import zlib buf = zlib.compress(buf) @@ -35,7 +34,7 @@ def make_doc_header(target, source, env): g.write("static const int _doc_data_uncompressed_size = " + str(decomp_size) + ";\n") g.write("static const unsigned char _doc_data_compressed[] = {\n") for i in range(len(buf)): - g.write("\t" + byte_to_str(buf[i]) + ",\n") + g.write("\t" + str(buf[i]) + ",\n") g.write("};\n") g.write("#endif") @@ -47,7 +46,7 @@ def make_fonts_header(target, source, env): dst = target[0] - g = open_utf8(dst, "w") + g = open(dst, "w", encoding="utf-8") g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") g.write("#ifndef _EDITOR_FONTS_H\n") @@ -64,7 +63,7 @@ def make_fonts_header(target, source, env): g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n") g.write("static const unsigned char _font_" + name + "[] = {\n") for j in range(len(buf)): - g.write("\t" + byte_to_str(buf[j]) + ",\n") + g.write("\t" + str(buf[j]) + ",\n") g.write("};\n") @@ -77,7 +76,7 @@ def make_translations_header(target, source, env, category): dst = target[0] - g = open_utf8(dst, "w") + g = open(dst, "w", encoding="utf-8") g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper())) @@ -98,7 +97,7 @@ def make_translations_header(target, source, env, category): g.write("static const unsigned char _{}_translation_{}_compressed[] = {{\n".format(category, name)) for j in range(len(buf)): - g.write("\t" + byte_to_str(buf[j]) + ",\n") + g.write("\t" + str(buf[j]) + ",\n") g.write("};\n") diff --git a/editor/icons/editor_icons_builders.py b/editor/icons/editor_icons_builders.py index ea2c2e57d12..a00f21c265b 100644 --- a/editor/icons/editor_icons_builders.py +++ b/editor/icons/editor_icons_builders.py @@ -3,9 +3,10 @@ All such functions are invoked in a subprocess on Windows to prevent build flakiness. """ + import os +from io import StringIO from platform_methods import subprocess_main -from compat import StringIO def make_editor_icons_action(target, source, env): diff --git a/main/main_builders.py b/main/main_builders.py index c48aaaa5725..e24070ccc38 100644 --- a/main/main_builders.py +++ b/main/main_builders.py @@ -4,7 +4,6 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki """ from platform_methods import subprocess_main -from compat import byte_to_str from collections import OrderedDict @@ -22,7 +21,7 @@ def make_splash(target, source, env): g.write('static const Color boot_splash_bg_color = Color(0.14, 0.14, 0.14);\n') g.write("static const unsigned char boot_splash_png[] = {\n") for i in range(len(buf)): - g.write(byte_to_str(buf[i]) + ",\n") + g.write(str(buf[i]) + ",\n") g.write("};\n") g.write("#endif") @@ -41,7 +40,7 @@ def make_splash_editor(target, source, env): g.write('static const Color boot_splash_editor_bg_color = Color(0.14, 0.14, 0.14);\n') g.write("static const unsigned char boot_splash_editor_png[] = {\n") for i in range(len(buf)): - g.write(byte_to_str(buf[i]) + ",\n") + g.write(str(buf[i]) + ",\n") g.write("};\n") g.write("#endif") @@ -59,7 +58,7 @@ def make_app_icon(target, source, env): g.write("#define APP_ICON_H\n") g.write("static const unsigned char app_icon_png[] = {\n") for i in range(len(buf)): - g.write(byte_to_str(buf[i]) + ",\n") + g.write(str(buf[i]) + ",\n") g.write("};\n") g.write("#endif") diff --git a/methods.py b/methods.py index 80c89d8c163..dc82c973618 100644 --- a/methods.py +++ b/methods.py @@ -2,12 +2,11 @@ import os import re import glob import subprocess -from compat import iteritems, isbasestring, decode_utf8 def add_source_files(self, sources, files, warn_duplicates=True): # Convert string to list of absolute paths (including expanding wildcard) - if isbasestring(files): + if isinstance(files, (str, bytes)): # Keep SCons project-absolute path as they are (no wildcard support) if files.startswith('#'): if '*' in files: @@ -240,7 +239,7 @@ def use_windows_spawn_fix(self, platform=None): cmdline = cmd + " " + newargs rv = 0 - env = {str(key): str(value) for key, value in iteritems(env)} + env = {str(key): str(value) for key, value in iter(env.items())} if len(cmdline) > 32000 and cmd.endswith("ar"): cmdline = cmd + " " + args[1] + " " + args[2] + " " for i in range(3, len(args)): @@ -530,7 +529,7 @@ def detect_darwin_sdk_path(platform, env): if not env[var_name]: try: - sdk_path = decode_utf8(subprocess.check_output(['xcrun', '--sdk', sdk_name, '--show-sdk-path']).strip()) + sdk_path = subprocess.check_output(['xcrun', '--sdk', sdk_name, '--show-sdk-path']).strip().decode("utf-8") if sdk_path: env[var_name] = sdk_path except (subprocess.CalledProcessError, OSError): @@ -540,7 +539,7 @@ def detect_darwin_sdk_path(platform, env): def is_vanilla_clang(env): if not using_clang(env): return False - version = decode_utf8(subprocess.check_output([env['CXX'], '--version']).strip()) + version = subprocess.check_output([env['CXX'], '--version']).strip().decode("utf-8") return not version.startswith("Apple") @@ -553,7 +552,7 @@ def get_compiler_version(env): # Not using -dumpversion as some GCC distros only return major, and # Clang used to return hardcoded 4.2.1: # https://reviews.llvm.org/D56803 try: - version = decode_utf8(subprocess.check_output([env.subst(env['CXX']), '--version']).strip()) + version = subprocess.check_output([env.subst(env['CXX']), '--version']).strip().decode("utf-8") except (subprocess.CalledProcessError, OSError): print("Couldn't parse CXX environment variable to infer compiler version.") return None diff --git a/modules/freetype/SCsub b/modules/freetype/SCsub index 7b66aa1c766..9e1853c4cda 100644 --- a/modules/freetype/SCsub +++ b/modules/freetype/SCsub @@ -3,8 +3,6 @@ Import('env') Import('env_modules') -from compat import isbasestring - env_freetype = env_modules.Clone() # Thirdparty source files @@ -93,7 +91,7 @@ if env['builtin_freetype']: # and then plain strings for system library. We insert between the two. inserted = False for idx, linklib in enumerate(env["LIBS"]): - if isbasestring(linklib): # first system lib such as "X11", otherwise SCons lib object + if isinstance(linklib, (str, bytes)): # first system lib such as "X11", otherwise SCons lib object env["LIBS"].insert(idx, lib) inserted = True break diff --git a/modules/mono/build_scripts/make_android_mono_config.py b/modules/mono/build_scripts/make_android_mono_config.py index 0afd939c579..4f5a4968910 100644 --- a/modules/mono/build_scripts/make_android_mono_config.py +++ b/modules/mono/build_scripts/make_android_mono_config.py @@ -1,7 +1,6 @@ def generate_compressed_config(config_src, output_dir): import os.path - from compat import byte_to_str # Source file with open(os.path.join(output_dir, 'android_mono_config.gen.cpp'), 'w') as cpp: @@ -16,7 +15,7 @@ def generate_compressed_config(config_src, output_dir): for i, buf_idx in enumerate(range(compr_size)): if i > 0: bytes_seq_str += ', ' - bytes_seq_str += byte_to_str(buf[buf_idx]) + bytes_seq_str += str(buf[buf_idx]) cpp.write('''/* THIS FILE IS GENERATED DO NOT EDIT */ #include "android_mono_config.h" diff --git a/modules/mono/build_scripts/mono_reg_utils.py b/modules/mono/build_scripts/mono_reg_utils.py index b2c48f0a61f..3bae11b1675 100644 --- a/modules/mono/build_scripts/mono_reg_utils.py +++ b/modules/mono/build_scripts/mono_reg_utils.py @@ -1,14 +1,9 @@ import os import platform -from compat import decode_utf8 - if os.name == 'nt': import sys - if sys.version_info < (3,): - import _winreg as winreg - else: - import winreg + import winreg def _reg_open_key(key, subkey): @@ -81,7 +76,7 @@ def find_msbuild_tools_path_reg(): lines = subprocess.check_output([vswhere] + vswhere_args).splitlines() for line in lines: - parts = decode_utf8(line).split(':', 1) + parts = line.decode("utf-8").split(':', 1) if len(parts) < 2 or parts[0] != 'installationPath': continue diff --git a/platform/SCsub b/platform/SCsub index 38bab59d74c..40cacce6744 100644 --- a/platform/SCsub +++ b/platform/SCsub @@ -1,7 +1,5 @@ #!/usr/bin/env python -from compat import open_utf8 - Import('env') env.platform_sources = [] @@ -21,7 +19,7 @@ reg_apis += '}\n\n' unreg_apis += '}\n' # NOTE: It is safe to generate this file here, since this is still execute serially -with open_utf8('register_platform_apis.gen.cpp', 'w') as f: +with open('register_platform_apis.gen.cpp', 'w', encoding="utf-8") as f: f.write(reg_apis_inc) f.write(reg_apis) f.write(unreg_apis) diff --git a/platform_methods.py b/platform_methods.py index 43002164275..eed76bc8a87 100644 --- a/platform_methods.py +++ b/platform_methods.py @@ -7,10 +7,7 @@ import subprocess # NOTE: The multiprocessing module is not compatible with SCons due to conflict on cPickle -if sys.version_info[0] < 3: - JSON_SERIALIZABLE_TYPES = (bool, int, long, float, basestring) -else: - JSON_SERIALIZABLE_TYPES = (bool, int, float, str) +JSON_SERIALIZABLE_TYPES = (bool, int, float, str) def run_in_subprocess(builder_function):