From c5bd2f9dce17f3e6bf7d9681243e2743633db6f0 Mon Sep 17 00:00:00 2001 From: Jiri Suchan Date: Tue, 23 Aug 2022 22:21:46 +0900 Subject: [PATCH] ci: add Python static analysis check via mypy --- .github/workflows/static_checks.yml | 6 +++- core/core_builders.py | 6 ++-- core/input/input_builders.py | 2 +- doc/tools/make_rst.py | 29 ++++++++----------- doc/translations/extract.py | 2 +- editor/editor_builders.py | 5 +--- editor/translations/extract.py | 7 +++-- gles3_builders.py | 10 ++++--- glsl_builders.py | 14 +++++---- methods.py | 4 +-- misc/scripts/mypy.ini | 11 +++++++ misc/scripts/mypy_check.sh | 6 ++++ .../mono/build_scripts/build_assemblies.py | 8 ++--- platform/android/detect.py | 9 ++++-- platform/ios/detect.py | 7 ++++- platform/linuxbsd/detect.py | 7 ++++- platform/macos/detect.py | 7 ++++- platform/uwp/detect.py | 11 +++++-- platform/web/detect.py | 6 +++- platform/windows/detect.py | 7 ++++- 20 files changed, 105 insertions(+), 59 deletions(-) create mode 100644 misc/scripts/mypy.ini create mode 100755 misc/scripts/mypy_check.sh diff --git a/.github/workflows/static_checks.yml b/.github/workflows/static_checks.yml index 5b4de06e9e6..d8951ddb788 100644 --- a/.github/workflows/static_checks.yml +++ b/.github/workflows/static_checks.yml @@ -27,7 +27,7 @@ jobs: sudo apt-get install -qq dos2unix recode clang-format-13 libxml2-utils python3-pip moreutils sudo update-alternatives --remove-all clang-format || true sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100 - sudo pip3 install black==22.3.0 pygments pytest + sudo pip3 install black==22.3.0 pygments pytest==7.1.2 mypy==0.971 - name: File formatting checks (file_format.sh) run: | @@ -41,6 +41,10 @@ jobs: run: | bash ./misc/scripts/black_format.sh + - name: Python scripts static analysis (mypy_check.sh) + run: | + bash ./misc/scripts/mypy_check.sh + - name: Python builders checks via pytest (pytest_builders.sh) run: | bash ./misc/scripts/pytest_builders.sh diff --git a/core/core_builders.py b/core/core_builders.py index b07daa80aea..b0a3b85d58c 100644 --- a/core/core_builders.py +++ b/core/core_builders.py @@ -2,6 +2,7 @@ All such functions are invoked in a subprocess on Windows to prevent build flakiness. """ +import zlib from platform_methods import subprocess_main @@ -33,7 +34,6 @@ def make_certs_header(target, source, env): g = open(dst, "w", encoding="utf-8") buf = f.read() decomp_size = len(buf) - import zlib # Use maximum zlib compression level to further reduce file size # (at the cost of initial build times). @@ -208,7 +208,7 @@ def make_license_header(target, source, env): from collections import OrderedDict - projects = OrderedDict() + projects: dict = OrderedDict() license_list = [] with open(src_copyright, "r", encoding="utf-8") as copyright_file: @@ -230,7 +230,7 @@ def make_license_header(target, source, env): part = {} reader.next_line() - data_list = [] + data_list: list = [] for project in iter(projects.values()): for part in project: part["file_index"] = len(data_list) diff --git a/core/input/input_builders.py b/core/input/input_builders.py index 16f125ff387..a7729c9af2e 100644 --- a/core/input/input_builders.py +++ b/core/input/input_builders.py @@ -16,7 +16,7 @@ def make_default_controller_mappings(target, source, env): g.write('#include "core/input/default_controller_mappings.h"\n') # ensure mappings have a consistent order - platform_mappings = OrderedDict() + platform_mappings: dict = OrderedDict() for src_path in source: with open(src_path, "r") as f: # read mapping file and skip header diff --git a/doc/tools/make_rst.py b/doc/tools/make_rst.py index a8569413ec3..492a438d9b7 100755 --- a/doc/tools/make_rst.py +++ b/doc/tools/make_rst.py @@ -526,7 +526,7 @@ def main() -> None: ) if os.path.exists(lang_file): try: - import polib + import polib # type: ignore except ImportError: print("Base template strings localization requires `polib`.") exit(1) @@ -739,9 +739,10 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: f.write(f"- {make_link(url, title)}\n\n") # Properties overview + ml: List[Tuple[Optional[str], ...]] = [] if len(class_def.properties) > 0: f.write(make_heading("Properties", "-")) - ml: List[Tuple[Optional[str], ...]] = [] + ml = [] for property_def in class_def.properties.values(): type_rst = property_def.type_name.to_rst(state) default = property_def.default_value @@ -757,7 +758,7 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: # Constructors, Methods, Operators overview if len(class_def.constructors) > 0: f.write(make_heading("Constructors", "-")) - ml: List[Tuple[Optional[str], ...]] = [] + ml = [] for method_list in class_def.constructors.values(): for m in method_list: ml.append(make_method_signature(class_def, m, "constructor", state)) @@ -765,7 +766,7 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: if len(class_def.methods) > 0: f.write(make_heading("Methods", "-")) - ml: List[Tuple[Optional[str], ...]] = [] + ml = [] for method_list in class_def.methods.values(): for m in method_list: ml.append(make_method_signature(class_def, m, "method", state)) @@ -773,7 +774,7 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: if len(class_def.operators) > 0: f.write(make_heading("Operators", "-")) - ml: List[Tuple[Optional[str], ...]] = [] + ml = [] for method_list in class_def.operators.values(): for m in method_list: ml.append(make_method_signature(class_def, m, "operator", state)) @@ -858,7 +859,7 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir: f.write(make_heading("Annotations", "-")) index = 0 - for method_list in class_def.annotations.values(): + for method_list in class_def.annotations.values(): # type: ignore for i, m in enumerate(method_list): if index != 0: f.write("----\n\n") @@ -1039,17 +1040,15 @@ def make_method_signature( ) -> Tuple[str, str]: ret_type = "" - is_method_def = isinstance(definition, MethodDef) - if is_method_def: + if isinstance(definition, MethodDef): ret_type = definition.return_type.to_rst(state) qualifiers = None - if is_method_def or isinstance(definition, AnnotationDef): + if isinstance(definition, (MethodDef, AnnotationDef)): qualifiers = definition.qualifiers out = "" - - if is_method_def and ref_type != "": + if isinstance(definition, MethodDef) and ref_type != "": if ref_type == "operator": op_name = definition.name.replace("<", "\\<") # So operator "<" gets correctly displayed. out += f":ref:`{op_name}` " @@ -1456,18 +1455,14 @@ def format_text_block( escape_post = True elif cmd.startswith("param"): - valid_context = ( - isinstance(context, MethodDef) - or isinstance(context, SignalDef) - or isinstance(context, AnnotationDef) - ) + valid_context = isinstance(context, (MethodDef, SignalDef, AnnotationDef)) if not valid_context: print_error( f'{state.current_class}.xml: Argument reference "{link_target}" used outside of method, signal, or annotation context in {context_name}.', state, ) else: - context_params: List[ParameterDef] = context.parameters + context_params: List[ParameterDef] = context.parameters # type: ignore found = False for param_def in context_params: if param_def.name == link_target: diff --git a/doc/translations/extract.py b/doc/translations/extract.py index 5708e0072d7..ce645436d9f 100644 --- a/doc/translations/extract.py +++ b/doc/translations/extract.py @@ -60,7 +60,7 @@ BASE_STRINGS = [ ## import sys -sys.modules["_elementtree"] = None +sys.modules["_elementtree"] = None # type: ignore import xml.etree.ElementTree as ET ## override the parser to get the line number diff --git a/editor/editor_builders.py b/editor/editor_builders.py index e73fbc6107d..696e3b64ec9 100644 --- a/editor/editor_builders.py +++ b/editor/editor_builders.py @@ -9,6 +9,7 @@ import shutil import subprocess import tempfile import uuid +import zlib from platform_methods import subprocess_main @@ -28,7 +29,6 @@ def make_doc_header(target, source, env): buf = (docbegin + buf + docend).encode("utf-8") decomp_size = len(buf) - import zlib # Use maximum zlib compression level to further reduce file size # (at the cost of initial build times). @@ -88,9 +88,6 @@ def make_translations_header(target, source, env, category): g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper())) g.write("#define _{}_TRANSLATIONS_H\n".format(category.upper())) - import zlib - import os.path - sorted_paths = sorted(source, key=lambda path: os.path.splitext(os.path.basename(path))[0]) msgfmt_available = shutil.which("msgfmt") is not None diff --git a/editor/translations/extract.py b/editor/translations/extract.py index 07026baee2c..cecdb3939d6 100755 --- a/editor/translations/extract.py +++ b/editor/translations/extract.py @@ -8,6 +8,7 @@ import re import shutil import subprocess import sys +from typing import Dict, Tuple class Message: @@ -42,7 +43,7 @@ class Message: return "\n".join(lines) -messages_map = {} # (id, context) -> Message. +messages_map: Dict[Tuple[str, str], Message] = {} # (id, context) -> Message. line_nb = False @@ -51,11 +52,11 @@ for arg in sys.argv[1:]: print("Enabling line numbers in the context locations.") line_nb = True else: - os.sys.exit("Non supported argument '" + arg + "'. Aborting.") + sys.exit("Non supported argument '" + arg + "'. Aborting.") if not os.path.exists("editor"): - os.sys.exit("ERROR: This script should be started from the root of the git repo.") + sys.exit("ERROR: This script should be started from the root of the git repo.") matches = [] diff --git a/gles3_builders.py b/gles3_builders.py index eafe503dd5b..84f11532e01 100644 --- a/gles3_builders.py +++ b/gles3_builders.py @@ -3,6 +3,10 @@ All such functions are invoked in a subprocess on Windows to prevent build flakiness. """ +import os.path + +from typing import Optional + from platform_methods import subprocess_main @@ -30,7 +34,7 @@ class GLES3HeaderStruct: self.specialization_values = [] -def include_file_in_gles3_header(filename, header_data, depth): +def include_file_in_gles3_header(filename: str, header_data: GLES3HeaderStruct, depth: int): fs = open(filename, "r") line = fs.readline() @@ -91,8 +95,6 @@ def include_file_in_gles3_header(filename, header_data, depth): while line.find("#include ") != -1: includeline = line.replace("#include ", "").strip()[1:-1] - import os.path - included_file = os.path.relpath(os.path.dirname(filename) + "/" + includeline) if not included_file in header_data.vertex_included_files and header_data.reading == "vertex": header_data.vertex_included_files += [included_file] @@ -182,7 +184,7 @@ def include_file_in_gles3_header(filename, header_data, depth): return header_data -def build_gles3_header(filename, include, class_suffix, header_data=None): +def build_gles3_header(filename: str, include: str, class_suffix: str, header_data: Optional[GLES3HeaderStruct] = None): header_data = header_data or GLES3HeaderStruct() include_file_in_gles3_header(filename, header_data, 0) diff --git a/glsl_builders.py b/glsl_builders.py index 8cb5807f21a..888f541cf4e 100644 --- a/glsl_builders.py +++ b/glsl_builders.py @@ -4,13 +4,15 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki """ import os.path +from typing import Optional, Iterable + from platform_methods import subprocess_main -def generate_inline_code(input_lines, insert_newline=True): +def generate_inline_code(input_lines: Iterable[str], insert_newline: bool = True): """Take header data and generate inline code - :param: list input_lines: values for shared inline code + :param: input_lines: values for shared inline code :return: str - generated inline value """ output = [] @@ -40,7 +42,7 @@ class RDHeaderStruct: self.compute_offset = 0 -def include_file_in_rd_header(filename, header_data, depth): +def include_file_in_rd_header(filename: str, header_data: RDHeaderStruct, depth: int) -> RDHeaderStruct: fs = open(filename, "r") line = fs.readline() @@ -112,7 +114,7 @@ def include_file_in_rd_header(filename, header_data, depth): return header_data -def build_rd_header(filename, header_data=None): +def build_rd_header(filename: str, header_data: Optional[RDHeaderStruct] = None) -> None: header_data = header_data or RDHeaderStruct() include_file_in_rd_header(filename, header_data, 0) @@ -171,7 +173,7 @@ class RAWHeaderStruct: self.code = "" -def include_file_in_raw_header(filename, header_data, depth): +def include_file_in_raw_header(filename: str, header_data: RAWHeaderStruct, depth: int) -> None: fs = open(filename, "r") line = fs.readline() @@ -191,7 +193,7 @@ def include_file_in_raw_header(filename, header_data, depth): fs.close() -def build_raw_header(filename, header_data=None): +def build_raw_header(filename: str, header_data: Optional[RAWHeaderStruct] = None): header_data = header_data or RAWHeaderStruct() include_file_in_raw_header(filename, header_data, 0) diff --git a/methods.py b/methods.py index 8604d388ce1..dadac37cb5d 100644 --- a/methods.py +++ b/methods.py @@ -1,6 +1,5 @@ import os import re -import sys import glob import subprocess from collections import OrderedDict @@ -663,7 +662,6 @@ def detect_visual_c_compiler_version(tools_env): if vc_x86_amd64_compiler_detection_index > -1 and ( vc_chosen_compiler_index == -1 or vc_chosen_compiler_index > vc_x86_amd64_compiler_detection_index ): - vc_chosen_compiler_index = vc_x86_amd64_compiler_detection_index vc_chosen_compiler_str = "x86_amd64" return vc_chosen_compiler_str @@ -781,7 +779,7 @@ def generate_vs_project(env, num_jobs): f'bin\\godot.windows.{config}{dev}.{plat_id}{f".{name}" if name else ""}.exe' for config in ModuleConfigs.CONFIGURATIONS for plat_id in ModuleConfigs.PLATFORM_IDS - for dev in ModuleConfig.DEV_SUFFIX + for dev in ModuleConfigs.DEV_SUFFIX ] self.arg_dict["cpppaths"] += ModuleConfigs.for_every_variant(env["CPPPATH"] + [includes]) self.arg_dict["cppdefines"] += ModuleConfigs.for_every_variant(env["CPPDEFINES"] + defines) diff --git a/misc/scripts/mypy.ini b/misc/scripts/mypy.ini new file mode 100644 index 00000000000..c1ea695ca5a --- /dev/null +++ b/misc/scripts/mypy.ini @@ -0,0 +1,11 @@ +[mypy] +ignore_missing_imports = true +disallow_any_generics = True +pretty = True +show_column_numbers = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True + +namespace_packages = True +explicit_package_bases = True diff --git a/misc/scripts/mypy_check.sh b/misc/scripts/mypy_check.sh new file mode 100755 index 00000000000..2a06486d679 --- /dev/null +++ b/misc/scripts/mypy_check.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -uo pipefail + +echo -e "Python: mypy static analysis..." +mypy --config-file=./misc/scripts/mypy.ini . diff --git a/modules/mono/build_scripts/build_assemblies.py b/modules/mono/build_scripts/build_assemblies.py index d78a9c7db8e..d28c3a0c3ac 100755 --- a/modules/mono/build_scripts/build_assemblies.py +++ b/modules/mono/build_scripts/build_assemblies.py @@ -5,6 +5,7 @@ import os.path import shlex import subprocess from dataclasses import dataclass +from typing import Optional, List def find_dotnet_cli(): @@ -150,10 +151,7 @@ def find_any_msbuild_tool(mono_prefix): return None -def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: [str] = None): - if msbuild_args is None: - msbuild_args = [] - +def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: Optional[List[str]] = None): using_msbuild_mono = False # Preference order: dotnet CLI > Standalone MSBuild > Mono's MSBuild @@ -169,7 +167,7 @@ def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: [str] = None): args += [sln] - if len(msbuild_args) > 0: + if msbuild_args: args += msbuild_args print("Running MSBuild: ", " ".join(shlex.quote(arg) for arg in args), flush=True) diff --git a/platform/android/detect.py b/platform/android/detect.py index f3e3f80dd5b..e541aa0373b 100644 --- a/platform/android/detect.py +++ b/platform/android/detect.py @@ -3,6 +3,11 @@ import sys import platform import subprocess +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + def is_active(): return True @@ -17,8 +22,6 @@ def can_build(): def get_opts(): - from SCons.Variables import BoolVariable, EnumVariable - return [ ("ANDROID_SDK_ROOT", "Path to the Android SDK", get_env_android_sdk_root()), ("ndk_platform", 'Target platform (android-, e.g. "android-24")', "android-24"), @@ -74,7 +77,7 @@ def install_ndk_if_needed(env): env["ANDROID_NDK_ROOT"] = get_android_ndk_root(env) -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_32", "x86_64", "arm32", "arm64"] if env["arch"] not in supported_arches: diff --git a/platform/ios/detect.py b/platform/ios/detect.py index 74561e9fc58..38e62134b52 100644 --- a/platform/ios/detect.py +++ b/platform/ios/detect.py @@ -2,6 +2,11 @@ import os import sys from methods import detect_darwin_sdk_path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + def is_active(): return True @@ -42,7 +47,7 @@ def get_flags(): ] -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_64", "arm64"] if env["arch"] not in supported_arches: diff --git a/platform/linuxbsd/detect.py b/platform/linuxbsd/detect.py index 92af7e2d750..dfde0d249c2 100644 --- a/platform/linuxbsd/detect.py +++ b/platform/linuxbsd/detect.py @@ -4,6 +4,11 @@ import sys from methods import get_compiler_version, using_gcc from platform_methods import detect_arch +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + def is_active(): return True @@ -55,7 +60,7 @@ def get_flags(): ] -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_32", "x86_64", "arm32", "arm64", "rv64", "ppc32", "ppc64"] if env["arch"] not in supported_arches: diff --git a/platform/macos/detect.py b/platform/macos/detect.py index 58d209cc7b6..511286d52ba 100644 --- a/platform/macos/detect.py +++ b/platform/macos/detect.py @@ -3,6 +3,11 @@ import sys from methods import detect_darwin_sdk_path from platform_methods import detect_arch +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + def is_active(): return True @@ -72,7 +77,7 @@ def get_mvk_sdk_path(): return os.path.join(os.path.join(dirname, ver_file), "MoltenVK/MoltenVK.xcframework/macos-arm64_x86_64/") -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_64", "arm64"] if env["arch"] not in supported_arches: diff --git a/platform/uwp/detect.py b/platform/uwp/detect.py index 2c5746cb060..64fe5bc4a2f 100644 --- a/platform/uwp/detect.py +++ b/platform/uwp/detect.py @@ -3,6 +3,11 @@ import os import sys from platform_methods import detect_arch +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + def is_active(): return True @@ -39,7 +44,7 @@ def get_flags(): ] -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_32", "x86_64", "arm32"] if env["arch"] not in supported_arches: @@ -83,7 +88,7 @@ def configure(env): env.AppendUnique(CCFLAGS=["/utf-8"]) # ANGLE - angle_root = os.getenv("ANGLE_SRC_PATH") + angle_root = os.environ["ANGLE_SRC_PATH"] env.Prepend(CPPPATH=[angle_root + "/include"]) jobs = str(env.GetOption("num_jobs")) angle_build_cmd = ( @@ -94,7 +99,7 @@ def configure(env): + " /p:Configuration=Release /p:Platform=" ) - if os.path.isfile(str(os.getenv("ANGLE_SRC_PATH")) + "/winrt/10/src/angle.sln"): + if os.path.isfile(f"{angle_root}/winrt/10/src/angle.sln"): env["build_angle"] = True ## Architecture diff --git a/platform/web/detect.py b/platform/web/detect.py index 77921847a88..08c1ff7b4ab 100644 --- a/platform/web/detect.py +++ b/platform/web/detect.py @@ -11,6 +11,10 @@ from emscripten_helpers import ( ) from methods import get_compiler_version from SCons.Util import WhereIs +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment def is_active(): @@ -60,7 +64,7 @@ def get_flags(): ] -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["wasm32"] if env["arch"] not in supported_arches: diff --git a/platform/windows/detect.py b/platform/windows/detect.py index b184da49e49..a5d8d0344b0 100644 --- a/platform/windows/detect.py +++ b/platform/windows/detect.py @@ -4,6 +4,11 @@ import subprocess import sys from platform_methods import detect_arch +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from SCons import Environment + # To match other platforms STACK_SIZE = 8388608 @@ -588,7 +593,7 @@ def configure_mingw(env): env.Append(BUILDERS={"RES": env.Builder(action=build_res_file, suffix=".o", src_suffix=".rc")}) -def configure(env): +def configure(env: "Environment"): # Validate arch. supported_arches = ["x86_32", "x86_64", "arm32", "arm64"] if env["arch"] not in supported_arches: