Merge pull request #96044 from darksylinc/matias-TheForge-hotfix2

Disable extra memory tracking by default

core/config/engine.cpp

@@ -263,6 +263,10 @@ bool Engine::is_generate_spirv_debug_info_enabled() const {
 	return generate_spirv_debug_info;
 }
 
+bool Engine::is_extra_gpu_memory_tracking_enabled() const {
+	return extra_gpu_memory_tracking;
+}
+
 void Engine::set_print_error_messages(bool p_enabled) {
 	CoreGlobals::print_error_enabled = p_enabled;
 }

core/config/engine.h

@@ -72,6 +72,7 @@ private:
 	bool abort_on_gpu_errors = false;
 	bool use_validation_layers = false;
 	bool generate_spirv_debug_info = false;
+	bool extra_gpu_memory_tracking = false;
 	int32_t gpu_idx = -1;
 
 	uint64_t _process_frames = 0;
@@ -181,6 +182,7 @@ public:
 	bool is_abort_on_gpu_errors_enabled() const;
 	bool is_validation_layers_enabled() const;
 	bool is_generate_spirv_debug_info_enabled() const;
+	bool is_extra_gpu_memory_tracking_enabled() const;
 	int32_t get_gpu_index() const;
 
 	void increment_frames_drawn();

doc/classes/RenderingDevice.xml

@@ -497,7 +497,7 @@
 			<return type="int" />
 			<description>
 				Returns how many allocations the GPU has performed for internal driver structures.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_device_allocs_by_object_type" qualifiers="const">
@@ -506,7 +506,7 @@
 			<description>
 				Same as [method get_device_allocation_count] but filtered for a given object type.
 				The type argument must be in range [code][0; get_tracked_object_type_count - 1][/code]. If [method get_tracked_object_type_count] is 0, then type argument is ignored and always returns 0.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_device_memory_by_object_type" qualifiers="const">
@@ -515,7 +515,7 @@
 			<description>
 				Same as [method get_device_total_memory] but filtered for a given object type.
 				The type argument must be in range [code][0; get_tracked_object_type_count - 1][/code]. If [method get_tracked_object_type_count] is 0, then type argument is ignored and always returns 0.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_device_name" qualifiers="const">
@@ -534,7 +534,7 @@
 			<return type="int" />
 			<description>
 				Returns how much bytes the GPU is using.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_device_vendor_name" qualifiers="const">
@@ -547,7 +547,7 @@
 			<return type="int" />
 			<description>
 				Returns how many allocations the GPU driver has performed for internal driver structures.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_driver_allocs_by_object_type" qualifiers="const">
@@ -556,7 +556,24 @@
 			<description>
 				Same as [method get_driver_allocation_count] but filtered for a given object type.
 				The type argument must be in range [code][0; get_tracked_object_type_count - 1][/code]. If [method get_tracked_object_type_count] is 0, then type argument is ignored and always returns 0.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
+			</description>
+		</method>
+		<method name="get_driver_and_device_memory_report" qualifiers="const">
+			<return type="String" />
+			<description>
+				Returns string report in CSV format using the following methods:
+				- [method get_tracked_object_name]
+				- [method get_tracked_object_type_count]
+				- [method get_driver_total_memory]
+				- [method get_driver_allocation_count]
+				- [method get_driver_memory_by_object_type]
+				- [method get_driver_allocs_by_object_type]
+				- [method get_device_total_memory]
+				- [method get_device_allocation_count]
+				- [method get_device_memory_by_object_type]
+				- [method get_device_allocs_by_object_type]
+				This is only used by Vulkan in debug builds. Godot must also be started with the [code]--extra-gpu-memory-tracking[/code] [url=$DOCS_URL/tutorials/editor/command_line_tutorial.html]command line argument[/url].
 			</description>
 		</method>
 		<method name="get_driver_memory_by_object_type" qualifiers="const">
@@ -565,7 +582,7 @@
 			<description>
 				Same as [method get_driver_total_memory] but filtered for a given object type.
 				The type argument must be in range [code][0; get_tracked_object_type_count - 1][/code]. If [method get_tracked_object_type_count] is 0, then type argument is ignored and always returns 0.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_driver_resource">
@@ -581,7 +598,7 @@
 			<return type="int" />
 			<description>
 				Returns how much bytes the GPU driver is using for internal driver structures.
-				This is only used by Vulkan in Debug builds and can return 0 when this information is not tracked or unknown.
+				This is only used by Vulkan in debug builds and can return 0 when this information is not tracked or unknown.
 			</description>
 		</method>
 		<method name="get_frame_delay" qualifiers="const">
@@ -614,14 +631,14 @@
 				- SWAPCHAIN_KHR
 				- COMMAND_POOL
 				Thus if e.g. [code]get_tracked_object_name(5)[/code] returns "COMMAND_POOL", then [code]get_device_memory_by_object_type(5)[/code] returns the bytes used by the GPU for command pools.
-				This is only used by Vulkan in Debug builds.
+				This is only used by Vulkan in debug builds. Godot must also be started with the [code]--extra-gpu-memory-tracking[/code] [url=$DOCS_URL/tutorials/editor/command_line_tutorial.html]command line argument[/url].
 			</description>
 		</method>
 		<method name="get_tracked_object_type_count" qualifiers="const">
 			<return type="int" />
 			<description>
 				Returns how many types of trackable objects are.
-				This is only used by Vulkan in Debug builds.
+				This is only used by Vulkan in debug builds. Godot must also be started with the [code]--extra-gpu-memory-tracking[/code] [url=$DOCS_URL/tutorials/editor/command_line_tutorial.html]command line argument[/url].
 			</description>
 		</method>
 		<method name="index_array_create">

drivers/vulkan/rendering_context_driver_vulkan.cpp

@@ -106,7 +106,7 @@ const char *RenderingContextDriverVulkan::get_tracked_object_name(uint32_t p_typ
 
 	return vkTrackedObjectTypeNames[p_type_index];
 #else
-	return "VK_TRACK_DRIVER_* disabled at build time";
+	return "VK_TRACK_*_MEMORY disabled at build time";
 #endif
 }
 
@@ -120,6 +120,8 @@ uint64_t RenderingContextDriverVulkan::get_tracked_object_type_count() const {
 RenderingContextDriverVulkan::VkTrackedObjectType vk_object_to_tracked_object(VkObjectType p_type) {
 	if (p_type > VK_OBJECT_TYPE_COMMAND_POOL && p_type != (VkObjectType)RenderingContextDriverVulkan::VK_TRACKED_OBJECT_TYPE_VMA) {
 		switch (p_type) {
+			case VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE:
+				return RenderingContextDriverVulkan::VK_TRACKED_OBJECT_DESCRIPTOR_UPDATE_TEMPLATE_KHR;
 			case VK_OBJECT_TYPE_SURFACE_KHR:
 				return RenderingContextDriverVulkan::VK_TRACKED_OBJECT_TYPE_SURFACE;
 			case VK_OBJECT_TYPE_SWAPCHAIN_KHR:
@@ -128,6 +130,9 @@ RenderingContextDriverVulkan::VkTrackedObjectType vk_object_to_tracked_object(Vk
 				return RenderingContextDriverVulkan::VK_TRACKED_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT;
 			case VK_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT:
 				return RenderingContextDriverVulkan::VK_TRACKED_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT;
+			case VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_KHR:
+			case VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_NV:
+				return RenderingContextDriverVulkan::VK_TRACKED_OBJECT_TYPE_ACCELERATION_STRUCTURE;
 			default:
 				_err_print_error(FUNCTION_STR, __FILE__, __LINE__, "Unknown VkObjectType enum value " + itos((uint32_t)p_type) + ".Please add it to VkTrackedObjectType, switch statement in "
 																																 "vk_object_to_tracked_object and get_tracked_object_name.",
@@ -229,6 +234,9 @@ VkAllocationCallbacks *RenderingContextDriverVulkan::get_allocation_callbacks(Vk
 #if !defined(VK_TRACK_DRIVER_MEMORY)
 	return nullptr;
 #else
+	if (!Engine::get_singleton()->is_extra_gpu_memory_tracking_enabled()) {
+		return nullptr;
+	}
 
 #ifdef _MSC_VER
 #define LAMBDA_VK_CALL_CONV

drivers/vulkan/rendering_context_driver_vulkan.h

@@ -170,10 +170,12 @@ public:
 
 #if defined(VK_TRACK_DRIVER_MEMORY) || defined(VK_TRACK_DEVICE_MEMORY)
 	enum VkTrackedObjectType{
-		VK_TRACKED_OBJECT_TYPE_SURFACE = VK_OBJECT_TYPE_COMMAND_POOL + 1,
+		VK_TRACKED_OBJECT_DESCRIPTOR_UPDATE_TEMPLATE_KHR = VK_OBJECT_TYPE_COMMAND_POOL + 1,
+		VK_TRACKED_OBJECT_TYPE_SURFACE,
 		VK_TRACKED_OBJECT_TYPE_SWAPCHAIN,
 		VK_TRACKED_OBJECT_TYPE_DEBUG_UTILS_MESSENGER_EXT,
 		VK_TRACKED_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT,
+		VK_TRACKED_OBJECT_TYPE_ACCELERATION_STRUCTURE,
 		VK_TRACKED_OBJECT_TYPE_VMA,
 		VK_TRACKED_OBJECT_TYPE_COUNT
 	};

drivers/vulkan/rendering_device_driver_vulkan.cpp

@@ -503,7 +503,9 @@ Error RenderingDeviceDriverVulkan::_initialize_device_extensions() {
 	}
 
 #if defined(VK_TRACK_DEVICE_MEMORY)
-	_register_requested_device_extension(VK_EXT_DEVICE_MEMORY_REPORT_EXTENSION_NAME, false);
+	if (Engine::get_singleton()->is_extra_gpu_memory_tracking_enabled()) {
+		_register_requested_device_extension(VK_EXT_DEVICE_MEMORY_REPORT_EXTENSION_NAME, false);
+	}
 #endif
 	_register_requested_device_extension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME, false);
 
@@ -5044,6 +5046,8 @@ void RenderingDeviceDriverVulkan::on_device_lost() const {
 	if (fault_info.pAddressInfos) {
 		memfree(fault_info.pAddressInfos);
 	}
+
+	_err_print_error(FUNCTION_STR, __FILE__, __LINE__, context_driver->get_driver_and_device_memory_report());
 }
 
 void RenderingDeviceDriverVulkan::print_lost_device_info() {

main/main.cpp

@@ -605,6 +605,9 @@ void Main::print_help(const char *p_binary) {
 	print_help_option("--gpu-abort", "Abort on graphics API usage errors (usually validation layer errors). May help see the problem if your system freezes.\n", CLI_OPTION_AVAILABILITY_TEMPLATE_DEBUG);
 #endif
 	print_help_option("--generate-spirv-debug-info", "Generate SPIR-V debug information. This allows source-level shader debugging with RenderDoc.\n");
+#if defined(DEBUG_ENABLED) || defined(DEV_ENABLED)
+	print_help_option("--extra-gpu-memory-tracking", "Enables additional memory tracking (see class reference for `RenderingDevice.get_driver_and_device_memory_report()` and linked methods). Currently only implemented for Vulkan. Enabling this feature may cause crashes on some systems due to buggy drivers or bugs in the Vulkan Loader. See https://github.com/godotengine/godot/issues/95967\n");
+#endif
 	print_help_option("--remote-debug <uri>", "Remote debug (<protocol>://<host/IP>[:<port>], e.g. tcp://127.0.0.1:6007).\n");
 	print_help_option("--single-threaded-scene", "Force scene tree to run in single-threaded mode. Sub-thread groups are disabled and run on the main thread.\n");
 #if defined(DEBUG_ENABLED)
@@ -1204,6 +1207,8 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 #endif
 		} else if (arg == "--generate-spirv-debug-info") {
 			Engine::singleton->generate_spirv_debug_info = true;
+		} else if (arg == "--extra-gpu-memory-tracking") {
+			Engine::singleton->extra_gpu_memory_tracking = true;
 		} else if (arg == "--tablet-driver") {
 			if (N) {
 				tablet_driver = N->get();

servers/rendering/rendering_context_driver.cpp

@@ -84,6 +84,51 @@ void RenderingContextDriver::window_destroy(DisplayServer::WindowID p_window) {
 	window_surface_map.erase(p_window);
 }
 
+String RenderingContextDriver::get_driver_and_device_memory_report() const {
+	String report;
+
+	const uint32_t num_tracked_obj_types = static_cast<uint32_t>(get_tracked_object_type_count());
+
+	report += "=== Driver Memory Report ===";
+
+	report += "\nLaunch with --extra-gpu-memory-tracking and build with "
+			  "DEBUG_ENABLED for this functionality to work.";
+	report += "\nDevice memory may be unavailable if the API does not support it"
+			  "(e.g. VK_EXT_device_memory_report is unsupported).";
+	report += "\n";
+
+	report += "\nTotal Driver Memory:";
+	report += String::num_real(double(get_driver_total_memory()) / (1024.0 * 1024.0));
+	report += " MB";
+	report += "\nTotal Driver Num Allocations: ";
+	report += String::num_uint64(get_driver_allocation_count());
+
+	report += "\nTotal Device Memory:";
+	report += String::num_real(double(get_device_total_memory()) / (1024.0 * 1024.0));
+	report += " MB";
+	report += "\nTotal Device Num Allocations: ";
+	report += String::num_uint64(get_device_allocation_count());
+
+	report += "\n\nMemory use by object type (CSV format):";
+	report += "\n\nCategory; Driver memory in MB; Driver Allocation Count; "
+			  "Device memory in MB; Device Allocation Count";
+
+	for (uint32_t i = 0u; i < num_tracked_obj_types; ++i) {
+		report += "\n";
+		report += get_tracked_object_name(i);
+		report += ";";
+		report += String::num_real(double(get_driver_memory_by_object_type(i)) / (1024.0 * 1024.0));
+		report += ";";
+		report += String::num_uint64(get_driver_allocs_by_object_type(i));
+		report += ";";
+		report += String::num_real(double(get_device_memory_by_object_type(i)) / (1024.0 * 1024.0));
+		report += ";";
+		report += String::num_uint64(get_device_allocs_by_object_type(i));
+	}
+
+	return report;
+}
+
 const char *RenderingContextDriver::get_tracked_object_name(uint32_t p_type_index) const {
 	return "Tracking Unsupported by API";
 }

servers/rendering/rendering_context_driver.h

@@ -102,6 +102,8 @@ public:
 	virtual void surface_destroy(SurfaceID p_surface) = 0;
 	virtual bool is_debug_utils_enabled() const = 0;
 
+	String get_driver_and_device_memory_report() const;
+
 	virtual const char *get_tracked_object_name(uint32_t p_type_index) const;
 	virtual uint64_t get_tracked_object_type_count() const;
 

servers/rendering/rendering_device.cpp

@@ -5723,6 +5723,10 @@ uint64_t RenderingDevice::get_driver_resource(DriverResource p_resource, RID p_r
 	return driver->get_resource_native_handle(p_resource, driver_id);
 }
 
+String RenderingDevice::get_driver_and_device_memory_report() const {
+	return context->get_driver_and_device_memory_report();
+}
+
 String RenderingDevice::get_tracked_object_name(uint32_t p_type_index) const {
 	return context->get_tracked_object_name(p_type_index);
 }
@@ -6077,6 +6081,7 @@ void RenderingDevice::_bind_methods() {
 
 	ClassDB::bind_method(D_METHOD("get_perf_report"), &RenderingDevice::get_perf_report);
 
+	ClassDB::bind_method(D_METHOD("get_driver_and_device_memory_report"), &RenderingDevice::get_driver_and_device_memory_report);
 	ClassDB::bind_method(D_METHOD("get_tracked_object_name", "type_index"), &RenderingDevice::get_tracked_object_name);
 	ClassDB::bind_method(D_METHOD("get_tracked_object_type_count"), &RenderingDevice::get_tracked_object_type_count);
 	ClassDB::bind_method(D_METHOD("get_driver_total_memory"), &RenderingDevice::get_driver_total_memory);

servers/rendering/rendering_device.h

@@ -1417,6 +1417,8 @@ public:
 
 	uint64_t get_driver_resource(DriverResource p_resource, RID p_rid = RID(), uint64_t p_index = 0);
 
+	String get_driver_and_device_memory_report() const;
+
 	String get_tracked_object_name(uint32_t p_type_index) const;
 	uint64_t get_tracked_object_type_count() const;