Merge pull request #41332 from bruvzg/win_subsys_option

Revert #41164, add subsystem build option.

editor/editor_node.cpp

@@ -2625,6 +2625,7 @@ void EditorNode::_menu_option_confirm(int p_option, bool p_confirmed) {
 		case SETTINGS_TOGGLE_CONSOLE: {
 			bool was_visible = DisplayServer::get_singleton()->is_console_visible();
 			DisplayServer::get_singleton()->console_set_visible(!was_visible);
+			EditorSettings::get_singleton()->set_setting("interface/editor/hide_console_window", was_visible);
 		} break;
 		case EDITOR_SCREENSHOT: {
 			screenshot_timer->start();

editor/editor_settings.cpp

@@ -334,6 +334,7 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 	_initial_set("interface/editor/automatically_open_screenshots", true);
 	_initial_set("interface/editor/single_window_mode", false);
 	hints["interface/editor/single_window_mode"] = PropertyInfo(Variant::BOOL, "interface/editor/single_window_mode", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_RESTART_IF_CHANGED);
+	_initial_set("interface/editor/hide_console_window", false);
 	_initial_set("interface/editor/save_each_scene_on_quit", true); // Regression
 	_initial_set("interface/editor/quit_confirmation", true);
 

main/main.cpp

@@ -2292,6 +2292,13 @@ bool Main::start() {
 		}
 
 		if (project_manager || editor) {
+			if (DisplayServer::get_singleton()->has_feature(DisplayServer::FEATURE_CONSOLE_WINDOW)) {
+				// Hide console window if requested (Windows-only).
+				bool hide_console = EditorSettings::get_singleton()->get_setting(
+						"interface/editor/hide_console_window");
+				DisplayServer::get_singleton()->console_set_visible(!hide_console);
+			}
+
 			// Load SSL Certificates from Editor Settings (or builtin)
 			Crypto::load_default_certificates(EditorSettings::get_singleton()->get_setting(
 																					 "network/ssl/editor_ssl_certificates")

platform/windows/detect.py

@@ -65,6 +65,7 @@ def get_opts():
         # Vista support dropped after EOL due to GH-10243
         ("target_win_version", "Targeted Windows version, >= 0x0601 (Windows 7)", "0x0601"),
         EnumVariable("debug_symbols", "Add debugging symbols to release builds", "yes", ("yes", "no", "full")),
+        EnumVariable("windows_subsystem", "Windows subsystem", "gui", ("console", "gui")),
         BoolVariable("separate_debug_symbols", "Create a separate file containing debugging symbols", False),
         ("msvc_version", "MSVC version to use. Ignored if VCINSTALLDIR is set in shell env.", None),
         BoolVariable("use_mingw", "Use the Mingw compiler, even if MSVC is installed. Only used on Windows.", False),
@@ -177,6 +178,8 @@ def configure_msvc(env, manual_msvc_config):
     """Configure env to work with MSVC"""
 
     # Build type
+    if env["tests"]:
+        env["windows_subsystem"] = "console"
 
     if env["target"] == "release":
         if env["optimize"] == "speed":  # optimize for speed (default)
@@ -199,12 +202,15 @@ def configure_msvc(env, manual_msvc_config):
         env.AppendUnique(CPPDEFINES=["DEBUG_ENABLED"])
         env.Append(LINKFLAGS=["/DEBUG"])
 
-    env.Append(LINKFLAGS=["/SUBSYSTEM:WINDOWS"])
-
     if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
         env.AppendUnique(CCFLAGS=["/Z7"])
         env.AppendUnique(LINKFLAGS=["/DEBUG"])
 
+    if env["windows_subsystem"] == "gui":
+        env.Append(LINKFLAGS=["/SUBSYSTEM:WINDOWS"])
+    else:
+        env.Append(LINKFLAGS=["/SUBSYSTEM:CONSOLE"])
+
     ## Compile/link flags
 
     env.AppendUnique(CCFLAGS=["/MT", "/Gd", "/GR", "/nologo"])
@@ -302,6 +308,9 @@ def configure_mingw(env):
 
     ## Build type
 
+    if env["tests"]:
+        env["windows_subsystem"] = "console"
+
     if env["target"] == "release":
         env.Append(CCFLAGS=["-msse2"])
 
@@ -334,7 +343,10 @@ def configure_mingw(env):
         env.Append(CCFLAGS=["-g3"])
         env.Append(CPPDEFINES=["DEBUG_ENABLED"])
 
+    if env["windows_subsystem"] == "gui":
         env.Append(LINKFLAGS=["-Wl,--subsystem,windows"])
+    else:
+        env.Append(LINKFLAGS=["-Wl,--subsystem,console"])
 
     ## Compiler configuration
 

platform/windows/display_server_windows.cpp

@@ -1137,17 +1137,10 @@ void DisplayServerWindows::window_set_ime_position(const Point2i &p_pos, WindowI
 void DisplayServerWindows::console_set_visible(bool p_enabled) {
 	_THREAD_SAFE_METHOD_
 
-	if (console_visible == p_enabled) {
+	if (console_visible == p_enabled)
 		return;
-	}
-	if (p_enabled && GetConsoleWindow() == nullptr) { // Open new console if not attached.
-		own_console = true;
-		AllocConsole();
-	}
-	if (own_console) { // Note: Do not hide parent console.
 	ShowWindow(GetConsoleWindow(), p_enabled ? SW_SHOW : SW_HIDE);
 	console_visible = p_enabled;
-	}
 }
 
 bool DisplayServerWindows::is_console_visible() const {
@@ -3028,18 +3021,7 @@ DisplayServerWindows::DisplayServerWindows(const String &p_rendering_driver, Win
 	shift_mem = false;
 	control_mem = false;
 	meta_mem = false;
-
-	if (AttachConsole(ATTACH_PARENT_PROCESS)) {
-		FILE *_file = nullptr;
-		freopen_s(&_file, "CONOUT$", "w", stdout);
-		freopen_s(&_file, "CONOUT$", "w", stderr);
-		freopen_s(&_file, "CONIN$", "r", stdin);
-
-		printf("\n");
-		console_visible = true;
-	} else {
-		console_visible = false;
-	}
+	console_visible = IsWindowVisible(GetConsoleWindow());
 	hInstance = ((OS_Windows *)OS::get_singleton())->get_hinstance();
 
 	pressrc = 0;

platform/windows/display_server_windows.h

@@ -405,7 +405,6 @@ private:
 	bool drop_events = false;
 	bool in_dispatch_input_event = false;
 	bool console_visible = false;
-	bool own_console = false;
 
 	WNDCLASSEXW wc;