godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SCons: Remove `run_in_subprocess` dependency

This commit is contained in:
Thaddeus Crews 2024-03-11 13:05:37 -05:00
parent f040a351c2
commit 5a6e3cbcb0
No known key found for this signature in database
GPG Key ID: 62181B86FE9E5D84
22 changed files with 84 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
SConstruct
View File

@ -1038,3 +1038,13 @@ def print_elapsed_time():
atexit.register(print_elapsed_time)
def purge_flaky_files():
for build_failure in GetBuildFailures():
path = build_failure.node.abspath
if os.path.isfile(path):
os.remove(path)
atexit.register(purge_flaky_files)

29
core/core_builders.py
View File

@ -1,12 +1,7 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import zlib
from platform_methods import subprocess_main
def escape_string(s):
def charcode_to_c_escapes(c):
@ -29,8 +24,8 @@ def escape_string(s):
def make_certs_header(target, source, env):
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "rb") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
buf = f.read()
decomp_size = len(buf)
@ -72,8 +67,8 @@ def make_authors_header(target, source, env):
"AUTHORS_DEVELOPERS",
]
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "r", encoding="utf-8") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
g.write("#ifndef AUTHORS_GEN_H\n")
@ -129,8 +124,8 @@ def make_donors_header(target, source, env):
"DONORS_MEMBERS_GOLD",
]
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "r", encoding="utf-8") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
g.write("#ifndef DONORS_GEN_H\n")
@ -165,9 +160,9 @@ def make_donors_header(target, source, env):
def make_license_header(target, source, env):
src_copyright = source[0]
src_license = source[1]
dst = target[0]
src_copyright = str(source[0])
src_license = str(source[1])
dst = str(target[0])
class LicenseReader:
def __init__(self, license_file):
@ -318,7 +313,3 @@ def make_license_header(target, source, env):
f.write("};\n\n")
f.write("#endif // LICENSE_GEN_H\n")
if __name__ == "__main__":
subprocess_main(globals())

10
core/extension/make_interface_dumper.py
View File

@ -2,8 +2,8 @@ import zlib
def run(target, source, env):
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "rb") as f, open(dst, "w", encoding="utf-8", newline="\n") as g:
buf = f.read()
decomp_size = len(buf)
@ -53,9 +53,3 @@ class GDExtensionInterfaceDump {
#endif // GDEXTENSION_INTERFACE_DUMP_H
"""
)
if __name__ == "__main__":
from platform_methods import subprocess_main
subprocess_main(globals())

8
core/extension/make_wrappers.py
View File

@ -142,11 +142,5 @@ def run(target, source, env):
txt += "\n#endif\n"
with open(target[0], "w", encoding="utf-8", newline="\n") as f:
with open(str(target[0]), "w", encoding="utf-8", newline="\n") as f:
f.write(txt)
if __name__ == "__main__":
from platform_methods import subprocess_main
subprocess_main(globals())

14
core/input/input_builders.py
View File

@ -1,14 +1,10 @@
"""Functions used to generate source files during build time
"""Functions used to generate source files during build time"""
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
from platform_methods import subprocess_main
from collections import OrderedDict
def make_default_controller_mappings(target, source, env):
dst = target[0]
dst = str(target[0])
with open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
g.write('#include "core/typedefs.h"\n')
@ -17,7 +13,7 @@ def make_default_controller_mappings(target, source, env):
# ensure mappings have a consistent order
platform_mappings: dict = OrderedDict()
for src_path in source:
with open(src_path, "r") as f:
with open(str(src_path), "r") as f:
# read mapping file and skip header
mapping_file_lines = f.readlines()[2:]
@ -61,7 +57,3 @@ def make_default_controller_mappings(target, source, env):
g.write("#endif\n")
g.write("\tnullptr\n};\n")
if __name__ == "__main__":
subprocess_main(globals())

8
core/object/make_virtuals.py
View File

@ -201,11 +201,5 @@ def run(target, source, env):
txt += "#endif // GDVIRTUAL_GEN_H\n"
with open(target[0], "w", encoding="utf-8", newline="\n") as f:
with open(str(target[0]), "w", encoding="utf-8", newline="\n") as f:
f.write(txt)
if __name__ == "__main__":
from platform_methods import subprocess_main
subprocess_main(globals())

18
editor/editor_builders.py
View File

@ -1,8 +1,4 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
import os.path
@ -11,16 +7,16 @@ import subprocess
import tempfile
import uuid
import zlib
from platform_methods import subprocess_main
def make_doc_header(target, source, env):
dst = target[0]
dst = str(target[0])
with open(dst, "w", encoding="utf-8", newline="\n") as g:
buf = ""
docbegin = ""
docend = ""
for src in source:
src = str(src)
if not src.endswith(".xml"):
continue
with open(src, "r", encoding="utf-8") as f:
@ -49,14 +45,14 @@ def make_doc_header(target, source, env):
def make_translations_header(target, source, env, category):
dst = target[0]
dst = str(target[0])
with open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper()))
g.write("#define _{}_TRANSLATIONS_H\n".format(category.upper()))
sorted_paths = sorted(source, key=lambda path: os.path.splitext(os.path.basename(path))[0])
sorted_paths = sorted([str(x) for x in source], key=lambda path: os.path.splitext(os.path.basename(path))[0])
msgfmt_available = shutil.which("msgfmt") is not None
@ -139,7 +135,3 @@ def make_doc_translations_header(target, source, env):
def make_extractable_translations_header(target, source, env):
make_translations_header(target, source, env, "extractable")
if __name__ == "__main__":
subprocess_main(globals())

14
editor/icons/editor_icons_builders.py
View File

@ -1,18 +1,12 @@
"""
Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from io import StringIO
from platform_methods import subprocess_main
# See also `scene/theme/icons/default_theme_icons_builders.py`.
def make_editor_icons_action(target, source, env):
dst = target[0]
dst = str(target[0])
svg_icons = source
with StringIO() as icons_string, StringIO() as s:
@ -85,7 +79,3 @@ def make_editor_icons_action(target, source, env):
with open(dst, "w", encoding="utf-8", newline="\n") as f:
f.write(s.getvalue())
if __name__ == "__main__":
subprocess_main(globals())

14
editor/template_builders.py
View File

@ -1,10 +1,7 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from io import StringIO
from platform_methods import subprocess_main
def parse_template(inherits, source, delimiter):
@ -53,7 +50,7 @@ def parse_template(inherits, source, delimiter):
def make_templates(target, source, env):
dst = target[0]
dst = str(target[0])
with StringIO() as s:
s.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n\n")
s.write("#ifndef _CODE_TEMPLATES_H\n")
@ -63,7 +60,7 @@ def make_templates(target, source, env):
delimiter = "#" # GDScript single line comment delimiter by default.
if source:
ext = os.path.splitext(source[0])[1]
ext = os.path.splitext(str(source[0]))[1]
if ext == ".cs":
delimiter = "//"
@ -71,6 +68,7 @@ def make_templates(target, source, env):
number_of_templates = 0
for filepath in source:
filepath = str(filepath)
node_name = os.path.basename(os.path.dirname(filepath))
parsed_template = parse_template(node_name, filepath, delimiter)
parsed_template_string += "\t" + parsed_template
@ -89,7 +87,3 @@ def make_templates(target, source, env):
with open(dst, "w", encoding="utf-8", newline="\n") as f:
f.write(s.getvalue())
if __name__ == "__main__":
subprocess_main(globals())

19
editor/themes/editor_theme_builders.py
View File

@ -1,16 +1,10 @@
"""
Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from platform_methods import subprocess_main
def make_fonts_header(target, source, env):
dst = target[0]
dst = str(target[0])
with open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
@ -19,10 +13,11 @@ def make_fonts_header(target, source, env):
# Saving uncompressed, since FreeType will reference from memory pointer.
for i in range(len(source)):
with open(source[i], "rb") as f:
file = str(source[i])
with open(file, "rb") as f:
buf = f.read()
name = os.path.splitext(os.path.basename(source[i]))[0]
name = os.path.splitext(os.path.basename(file))[0]
g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n")
g.write("static const unsigned char _font_" + name + "[] = {\n")
@ -32,7 +27,3 @@ def make_fonts_header(target, source, env):
g.write("};\n")
g.write("#endif")
if __name__ == "__main__":
subprocess_main(globals())

12
gles3_builders.py
View File

@ -1,15 +1,9 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os.path
from typing import Optional
from platform_methods import subprocess_main
class GLES3HeaderStruct:
def __init__(self):
@ -600,7 +594,3 @@ def build_gles3_header(
def build_gles3_headers(target, source, env):
for x in source:
build_gles3_header(str(x), include="drivers/gles3/shader_gles3.h", class_suffix="GLES3")
if __name__ == "__main__":
subprocess_main(globals())

12
glsl_builders.py
View File

@ -1,14 +1,8 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os.path
from typing import Optional, Iterable
from platform_methods import subprocess_main
def generate_inline_code(input_lines: Iterable[str], insert_newline: bool = True):
"""Take header data and generate inline code
@ -227,7 +221,3 @@ static const char {out_file_base}[] = {{
def build_raw_headers(target, source, env):
for x in source:
build_raw_header(filename=str(x))
if __name__ == "__main__":
subprocess_main(globals())

24
main/main_builders.py
View File

@ -1,15 +1,9 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
from platform_methods import subprocess_main
"""Functions used to generate source files during build time"""
def make_splash(target, source, env):
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "rb") as f:
buf = f.read()
@ -28,8 +22,8 @@ def make_splash(target, source, env):
def make_splash_editor(target, source, env):
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "rb") as f:
buf = f.read()
@ -49,8 +43,8 @@ def make_splash_editor(target, source, env):
def make_app_icon(target, source, env):
src = source[0]
dst = target[0]
src = str(source[0])
dst = str(target[0])
with open(src, "rb") as f:
buf = f.read()
@ -64,7 +58,3 @@ def make_app_icon(target, source, env):
g.write(str(buf[i]) + ",\n")
g.write("};\n")
g.write("#endif")
if __name__ == "__main__":
subprocess_main(globals())

8
methods.py
View File

@ -826,14 +826,10 @@ def CommandNoCache(env, target, sources, command, **args):
return result
def Run(env, function, subprocess=True):
def Run(env, function):
from SCons.Script import Action
from platform_methods import run_in_subprocess
if not subprocess:
return Action(function, "$GENCOMSTR")
else:
return Action(run_in_subprocess(function), "$GENCOMSTR")
return Action(function, "$GENCOMSTR")
def detect_darwin_sdk_path(platform, env):

12
modules/SCsub
View File

@ -17,11 +17,7 @@ env.Depends("modules_enabled.gen.h", Value(env.module_list))
env.CommandNoCache(
"modules_enabled.gen.h",
Value(env.module_list),
env.Run(
modules_builders.generate_modules_enabled,
# NOTE: No need to run in subprocess since this is still executed serially.
subprocess=False,
),
env.Run(modules_builders.generate_modules_enabled),
)
@ -55,11 +51,7 @@ if env["tests"]:
env.CommandNoCache(
"modules_tests.gen.h",
test_headers,
env.Run(
modules_builders.generate_modules_tests,
# NOTE: No need to run in subprocess since this is still executed serially.
subprocess=False,
),
env.Run(modules_builders.generate_modules_tests),
)
# libmodules.a with only register_module_types.

11
modules/modules_builders.py
View File

@ -1,9 +1,4 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
from platform_methods import subprocess_main
"""Functions used to generate source files during build time"""
def generate_modules_enabled(target, source, env):
@ -18,7 +13,3 @@ def generate_modules_tests(target, source, env):
with open(target[0].path, "w", encoding="utf-8", newline="\n") as f:
for header in source:
f.write('#include "%s"\n' % (os.path.normpath(header.path)))
if __name__ == "__main__":
subprocess_main(globals())

18
platform/linuxbsd/platform_linuxbsd_builders.py
View File

@ -1,18 +1,10 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from platform_methods import subprocess_main
def make_debug_linuxbsd(target, source, env):
os.system("objcopy --only-keep-debug {0} {0}.debugsymbols".format(target[0]))
os.system("strip --strip-debug --strip-unneeded {0}".format(target[0]))
os.system("objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(target[0]))
if __name__ == "__main__":
subprocess_main(globals())
dst = str(target[0])
os.system("objcopy --only-keep-debug {0} {0}.debugsymbols".format(dst))
os.system("strip --strip-debug --strip-unneeded {0}".format(dst))
os.system("objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(dst))

18
platform/macos/platform_macos_builders.py
View File

@ -1,22 +1,14 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from platform_methods import subprocess_main
def make_debug_macos(target, source, env):
dst = str(target[0])
if env["macports_clang"] != "no":
mpprefix = os.environ.get("MACPORTS_PREFIX", "/opt/local")
mpclangver = env["macports_clang"]
os.system(mpprefix + "/libexec/llvm-" + mpclangver + "/bin/llvm-dsymutil {0} -o {0}.dSYM".format(target[0]))
os.system(mpprefix + "/libexec/llvm-" + mpclangver + "/bin/llvm-dsymutil {0} -o {0}.dSYM".format(dst))
else:
os.system("dsymutil {0} -o {0}.dSYM".format(target[0]))
os.system("strip -u -r {0}".format(target[0]))
if __name__ == "__main__":
subprocess_main(globals())
os.system("dsymutil {0} -o {0}.dSYM".format(dst))
os.system("strip -u -r {0}".format(dst))

26
platform/windows/platform_windows_builders.py
View File

@ -1,32 +1,24 @@
"""Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from detect import get_mingw_bin_prefix
from detect import try_cmd
from platform_methods import subprocess_main
def make_debug_mingw(target, source, env):
dst = str(target[0])
# Force separate debug symbols if executable size is larger than 1.9 GB.
if env["separate_debug_symbols"] or os.stat(target[0]).st_size >= 2040109465:
if env["separate_debug_symbols"] or os.stat(dst).st_size >= 2040109465:
mingw_bin_prefix = get_mingw_bin_prefix(env["mingw_prefix"], env["arch"])
if try_cmd("objcopy --version", env["mingw_prefix"], env["arch"]):
os.system(mingw_bin_prefix + "objcopy --only-keep-debug {0} {0}.debugsymbols".format(target[0]))
os.system(mingw_bin_prefix + "objcopy --only-keep-debug {0} {0}.debugsymbols".format(dst))
else:
os.system("objcopy --only-keep-debug {0} {0}.debugsymbols".format(target[0]))
os.system("objcopy --only-keep-debug {0} {0}.debugsymbols".format(dst))
if try_cmd("strip --version", env["mingw_prefix"], env["arch"]):
os.system(mingw_bin_prefix + "strip --strip-debug --strip-unneeded {0}".format(target[0]))
os.system(mingw_bin_prefix + "strip --strip-debug --strip-unneeded {0}".format(dst))
else:
os.system("strip --strip-debug --strip-unneeded {0}".format(target[0]))
os.system("strip --strip-debug --strip-unneeded {0}".format(dst))
if try_cmd("objcopy --version", env["mingw_prefix"], env["arch"]):
os.system(mingw_bin_prefix + "objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(target[0]))
os.system(mingw_bin_prefix + "objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(dst))
else:
os.system("objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(target[0]))
if __name__ == "__main__":
subprocess_main(globals())
os.system("objcopy --add-gnu-debuglink={0}.debugsymbols {0}".format(dst))

69
platform_methods.py
View File

@ -8,75 +8,6 @@ import subprocess
# NOTE: The multiprocessing module is not compatible with SCons due to conflict on cPickle
JSON_SERIALIZABLE_TYPES = (bool, int, float, str)
def run_in_subprocess(builder_function):
@functools.wraps(builder_function)
def wrapper(target, source, env):
# Convert SCons Node instances to absolute paths
target = [node.srcnode().abspath for node in target]
source = [node.srcnode().abspath for node in source]
# Short circuit on non-Windows platforms, no need to run in subprocess
if sys.platform not in ("win32", "cygwin"):
return builder_function(target, source, env)
# Identify module
module_name = builder_function.__module__
function_name = builder_function.__name__
module_path = sys.modules[module_name].__file__
if module_path.endswith(".pyc") or module_path.endswith(".pyo"):
module_path = module_path[:-1]
# Subprocess environment
subprocess_env = os.environ.copy()
subprocess_env["PYTHONPATH"] = os.pathsep.join([os.getcwd()] + sys.path)
# Keep only JSON serializable environment items
filtered_env = dict((key, value) for key, value in env.items() if isinstance(value, JSON_SERIALIZABLE_TYPES))
# Save parameters
args = (target, source, filtered_env)
data = dict(fn=function_name, args=args)
json_path = os.path.join(os.environ["TMP"], uuid.uuid4().hex + ".json")
with open(json_path, "wt", encoding="utf-8", newline="\n") as json_file:
json.dump(data, json_file, indent=2)
json_file_size = os.stat(json_path).st_size
if env["verbose"]:
print(
"Executing builder function in subprocess: "
"module_path=%r, parameter_file=%r, parameter_file_size=%r, target=%r, source=%r"
% (module_path, json_path, json_file_size, target, source)
)
try:
exit_code = subprocess.call([sys.executable, module_path, json_path], env=subprocess_env)
finally:
try:
os.remove(json_path)
except OSError as e:
# Do not fail the entire build if it cannot delete a temporary file
print(
"WARNING: Could not delete temporary file: path=%r; [%s] %s" % (json_path, e.__class__.__name__, e)
)
# Must succeed
if exit_code:
raise RuntimeError(
"Failed to run builder function in subprocess: module_path=%r; data=%r" % (module_path, data)
)
return wrapper
def subprocess_main(namespace):
with open(sys.argv[1]) as json_file:
data = json.load(json_file)
fn = namespace[data["fn"]]
fn(*data["args"])
# CPU architecture options.
architectures = ["x86_32", "x86_64", "arm32", "arm64", "rv64", "ppc32", "ppc64", "wasm32"]

19
scene/theme/default_theme_builders.py
View File

@ -1,17 +1,11 @@
"""
Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
import os.path
from platform_methods import subprocess_main
def make_fonts_header(target, source, env):
dst = target[0]
dst = str(target[0])
with open(dst, "w", encoding="utf-8", newline="\n") as g:
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
@ -20,10 +14,11 @@ def make_fonts_header(target, source, env):
# Saving uncompressed, since FreeType will reference from memory pointer.
for i in range(len(source)):
with open(source[i], "rb") as f:
file = str(source[i])
with open(file, "rb") as f:
buf = f.read()
name = os.path.splitext(os.path.basename(source[i]))[0]
name = os.path.splitext(os.path.basename(file))[0]
g.write("static const int _font_" + name + "_size = " + str(len(buf)) + ";\n")
g.write("static const unsigned char _font_" + name + "[] = {\n")
@ -33,7 +28,3 @@ def make_fonts_header(target, source, env):
g.write("};\n")
g.write("#endif")
if __name__ == "__main__":
subprocess_main(globals())

16
scene/theme/icons/default_theme_icons_builders.py
View File

@ -1,19 +1,13 @@
"""
Functions used to generate source files during build time
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
"""Functions used to generate source files during build time"""
import os
from io import StringIO
from platform_methods import subprocess_main
# See also `editor/icons/editor_icons_builders.py`.
def make_default_theme_icons_action(target, source, env):
dst = target[0]
svg_icons = source
dst = str(target[0])
svg_icons = [str(x) for x in source]
with StringIO() as icons_string, StringIO() as s:
for f in svg_icons:
@ -65,7 +59,3 @@ def make_default_theme_icons_action(target, source, env):
with open(dst, "w", encoding="utf-8", newline="\n") as f:
f.write(s.getvalue())
if __name__ == "__main__":
subprocess_main(globals())