From 5e6adb4a2dd947432e59ca00b6d046a68c534e10 Mon Sep 17 00:00:00 2001 From: David Nikdel Date: Wed, 6 Mar 2024 12:14:21 -0500 Subject: [PATCH] Merge uid_cache.bin and global_script_class_cache.cfg after mounting PCKs fixes godotengine#82061 fixes godotengine#61556 Also, distinguish between main pack and DLC packs. It's desirable to downloaded content to be as small as possible. This change avoids bloating non-main pack files with new versions of resources that are all read on startup and never used again. They have no effect if loaded after startup. - project.godot/project.binary file - extension_list.cfg - app icon and boot_splash - .ico and .icns files (these can still be opted in for DLC by listing them explicitly in the include filter) --- core/config/project_settings.cpp | 23 +++++++ core/config/project_settings.h | 1 + core/io/resource_uid.cpp | 6 +- core/io/resource_uid.h | 2 +- core/object/script_language.cpp | 24 +++++-- editor/debugger/editor_file_server.cpp | 2 +- editor/editor_node.cpp | 3 + editor/export/editor_export_platform.cpp | 71 +++++++++++++-------- editor/export/editor_export_platform.h | 5 +- editor/export/editor_export_platform_pc.cpp | 2 +- editor/export/project_export.cpp | 2 + main/main.cpp | 2 +- platform/android/export/export_plugin.cpp | 10 +-- platform/ios/export/export_plugin.cpp | 2 +- platform/macos/export/export_plugin.cpp | 2 +- platform/web/export/export_plugin.cpp | 2 +- 16 files changed, 112 insertions(+), 47 deletions(-) diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index f5baf1a27ee..ce7fa1074b5 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -38,6 +38,8 @@ #include "core/io/file_access.h" #include "core/io/file_access_pack.h" #include "core/io/marshalls.h" +#include "core/io/resource_uid.h" +#include "core/object/script_language.h" #include "core/os/keyboard.h" #include "core/templates/rb_set.h" #include "core/variant/typed_array.h" @@ -478,6 +480,14 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f return false; } + if (project_loaded) { + // This pack may have declared new global classes (make sure they are picked up). + refresh_global_class_list(); + + // This pack may have defined new UIDs, make sure they are cached. + ResourceUID::get_singleton()->load_from_cache(false); + } + //if data.pck is found, all directory access will be from here DirAccess::make_default(DirAccess::ACCESS_RESOURCES); using_datapack = true; @@ -1189,6 +1199,19 @@ Variant ProjectSettings::get_setting(const String &p_setting, const Variant &p_d } } +void ProjectSettings::refresh_global_class_list() { + // This is called after mounting a new PCK file to pick up class changes. + is_global_class_list_loaded = false; // Make sure we read from the freshly mounted PCK. + Array script_classes = get_global_class_list(); + for (int i = 0; i < script_classes.size(); i++) { + Dictionary c = script_classes[i]; + if (!c.has("class") || !c.has("language") || !c.has("path") || !c.has("base")) { + continue; + } + ScriptServer::add_global_class(c["class"], c["base"], c["language"], c["path"]); + } +} + TypedArray ProjectSettings::get_global_class_list() { if (is_global_class_list_loaded) { return global_class_list; diff --git a/core/config/project_settings.h b/core/config/project_settings.h index 252b10bd3aa..1bad76acb10 100644 --- a/core/config/project_settings.h +++ b/core/config/project_settings.h @@ -154,6 +154,7 @@ public: void set_setting(const String &p_setting, const Variant &p_value); Variant get_setting(const String &p_setting, const Variant &p_default_value = Variant()) const; TypedArray get_global_class_list(); + void refresh_global_class_list(); void store_global_class_list(const Array &p_classes); String get_global_class_list_path() const; diff --git a/core/io/resource_uid.cpp b/core/io/resource_uid.cpp index edff3e1f149..c14121a53b3 100644 --- a/core/io/resource_uid.cpp +++ b/core/io/resource_uid.cpp @@ -169,14 +169,16 @@ Error ResourceUID::save_to_cache() { return OK; } -Error ResourceUID::load_from_cache() { +Error ResourceUID::load_from_cache(bool p_reset) { Ref f = FileAccess::open(get_cache_file(), FileAccess::READ); if (f.is_null()) { return ERR_CANT_OPEN; } MutexLock l(mutex); - unique_ids.clear(); + if (p_reset) { + unique_ids.clear(); + } uint32_t entry_count = f->get_32(); for (uint32_t i = 0; i < entry_count; i++) { diff --git a/core/io/resource_uid.h b/core/io/resource_uid.h index 22561c5c032..e56b89f6034 100644 --- a/core/io/resource_uid.h +++ b/core/io/resource_uid.h @@ -73,7 +73,7 @@ public: String get_id_path(ID p_id) const; void remove_id(ID p_id); - Error load_from_cache(); + Error load_from_cache(bool p_reset); Error save_to_cache(); Error update_cache(); diff --git a/core/object/script_language.cpp b/core/object/script_language.cpp index d358a8d2a0e..5885cf515f4 100644 --- a/core/object/script_language.cpp +++ b/core/object/script_language.cpp @@ -325,12 +325,24 @@ void ScriptServer::global_classes_clear() { void ScriptServer::add_global_class(const StringName &p_class, const StringName &p_base, const StringName &p_language, const String &p_path) { ERR_FAIL_COND_MSG(p_class == p_base || (global_classes.has(p_base) && get_global_class_native_base(p_base) == p_class), "Cyclic inheritance in script class."); - GlobalScriptClass g; - g.language = p_language; - g.path = p_path; - g.base = p_base; - global_classes[p_class] = g; - inheriters_cache_dirty = true; + GlobalScriptClass *existing = global_classes.getptr(p_class); + if (existing) { + // Update an existing class (only set dirty if something changed). + if (existing->base != p_base || existing->path != p_path || existing->language != p_language) { + existing->base = p_base; + existing->path = p_path; + existing->language = p_language; + inheriters_cache_dirty = true; + } + } else { + // Add new class. + GlobalScriptClass g; + g.language = p_language; + g.path = p_path; + g.base = p_base; + global_classes[p_class] = g; + inheriters_cache_dirty = true; + } } void ScriptServer::remove_global_class(const StringName &p_class) { diff --git a/editor/debugger/editor_file_server.cpp b/editor/debugger/editor_file_server.cpp index e84eb146363..428b5f4c261 100644 --- a/editor/debugger/editor_file_server.cpp +++ b/editor/debugger/editor_file_server.cpp @@ -201,7 +201,7 @@ void EditorFileServer::poll() { // Scan files to send. _scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files); // Add forced export files - Vector forced_export = EditorExportPlatform::get_forced_export_files(); + Vector forced_export = EditorExportPlatform::get_main_pack_forced_export_files(); for (int i = 0; i < forced_export.size(); i++) { _add_custom_file(forced_export[i], files_to_send, cached_files); } diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index 041784d7da4..5183bc4af35 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -969,6 +969,9 @@ void EditorNode::_fs_changed() { err = platform->export_zip(export_preset, export_defer.debug, export_path); } else if (export_path.ends_with(".pck")) { err = platform->export_pack(export_preset, export_defer.debug, export_path); + } else { + ERR_PRINT(vformat("Export path \"%s\" doesn't end with a supported extension.", export_path)); + err = FAILED; } } else { // Normal project export. String config_error; diff --git a/editor/export/editor_export_platform.cpp b/editor/export/editor_export_platform.cpp index b9dc52511e6..67714d50a0e 100644 --- a/editor/export/editor_export_platform.cpp +++ b/editor/export/editor_export_platform.cpp @@ -864,23 +864,19 @@ String EditorExportPlatform::_get_script_encryption_key(const Refget_script_encryption_key().to_lower(); } -Vector EditorExportPlatform::get_forced_export_files() { - Vector files; - - files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); +Vector EditorExportPlatform::get_main_pack_forced_export_files() { + // First, get files required by any PCK. + Vector files = get_forced_export_files(); String icon = GLOBAL_GET("application/config/icon"); - String splash = GLOBAL_GET("application/boot_splash/image"); if (!icon.is_empty() && FileAccess::exists(icon)) { files.push_back(icon); } - if (!splash.is_empty() && FileAccess::exists(splash) && icon != splash) { + + String splash = GLOBAL_GET("application/boot_splash/image"); + if (!splash.is_empty() && icon != splash && FileAccess::exists(splash)) { files.push_back(splash); } - String resource_cache_file = ResourceUID::get_cache_file(); - if (FileAccess::exists(resource_cache_file)) { - files.push_back(resource_cache_file); - } String extension_list_config_file = GDExtension::get_extension_list_config_file(); if (FileAccess::exists(extension_list_config_file)) { @@ -913,7 +909,19 @@ Vector EditorExportPlatform::get_forced_export_files() { return files; } -Error EditorExportPlatform::export_project_files(const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func) { +Vector EditorExportPlatform::get_forced_export_files() { + Vector files; + files.push_back(ProjectSettings::get_singleton()->get_global_class_list_path()); + + String resource_cache_file = ResourceUID::get_cache_file(); + if (FileAccess::exists(resource_cache_file)) { + files.push_back(resource_cache_file); + } + + return files; +} + +Error EditorExportPlatform::export_project_files(bool p_main_pack, const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func) { //figure out paths of files that will be exported HashSet paths; Vector path_remaps; @@ -960,9 +968,11 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - //add native icons to non-resource include list - _edit_filter_list(paths, String("*.icns"), false); - _edit_filter_list(paths, String("*.ico"), false); + if (p_main_pack) { + // Add native icons to non-resource include list. + _edit_filter_list(paths, String("*.icns"), false); + _edit_filter_list(paths, String("*.ico"), false); + } _edit_filter_list(paths, p_preset->get_include_filter(), false); _edit_filter_list(paths, p_preset->get_exclude_filter(), true); @@ -1389,7 +1399,12 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - Vector forced_export = get_forced_export_files(); + Vector forced_export; + if (p_main_pack) { + forced_export = get_main_pack_forced_export_files(); + } else { + forced_export = get_forced_export_files(); + } for (int i = 0; i < forced_export.size(); i++) { Vector array = FileAccess::get_file_as_bytes(forced_export[i]); err = p_func(p_udata, forced_export[i], array, idx, total, enc_in_filters, enc_ex_filters, key); @@ -1398,13 +1413,19 @@ Error EditorExportPlatform::export_project_files(const Ref & } } - String config_file = "project.binary"; - String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp" + config_file); - ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list); - Vector data = FileAccess::get_file_as_bytes(engine_cfb); - DirAccess::remove_file_or_error(engine_cfb); + if (p_main_pack) { + String config_file = "project.binary"; + String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().path_join("tmp" + config_file); + ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list); + Vector data = FileAccess::get_file_as_bytes(engine_cfb); + DirAccess::remove_file_or_error(engine_cfb); + err = p_func(p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key); + if (err != OK) { + return err; + } + } - return p_func(p_udata, "res://" + config_file, data, idx, total, enc_in_filters, enc_ex_filters, key); + return OK; } Error EditorExportPlatform::_add_shared_object(void *p_userdata, const SharedObject &p_so) { @@ -1533,7 +1554,7 @@ void EditorExportPlatform::zip_folder_recursive(zipFile &p_zip, const String &p_ da->list_dir_end(); } -Error EditorExportPlatform::save_pack(const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) { +Error EditorExportPlatform::save_pack(bool p_main_pack, const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) { EditorProgress ep("savepack", TTR("Packing"), 102, true); // Create the temporary export directory if it doesn't exist. @@ -1552,7 +1573,7 @@ Error EditorExportPlatform::save_pack(const Ref &p_preset, b pd.f = ftmp; pd.so_files = p_so_files; - Error err = export_project_files(p_preset, p_debug, _save_pack_file, &pd, _add_shared_object); + Error err = export_project_files(p_main_pack, p_preset, p_debug, _save_pack_file, &pd, _add_shared_object); // Close temp file. pd.f.unref(); @@ -1762,7 +1783,7 @@ Error EditorExportPlatform::save_zip(const Ref &p_preset, bo zd.ep = &ep; zd.zip = zip; - Error err = export_project_files(p_preset, p_debug, _save_zip_file, &zd); + Error err = export_project_files(false, p_preset, p_debug, _save_zip_file, &zd); if (err != OK && err != ERR_SKIP) { add_message(EXPORT_MESSAGE_ERROR, TTR("Save ZIP"), TTR("Failed to export project files.")); } @@ -1774,7 +1795,7 @@ Error EditorExportPlatform::save_zip(const Ref &p_preset, bo Error EditorExportPlatform::export_pack(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags); - return save_pack(p_preset, p_debug, p_path); + return save_pack(false, p_preset, p_debug, p_path); } Error EditorExportPlatform::export_zip(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { diff --git a/editor/export/editor_export_platform.h b/editor/export/editor_export_platform.h index 26e1f86c8b8..4c4e64b1a6c 100644 --- a/editor/export/editor_export_platform.h +++ b/editor/export/editor_export_platform.h @@ -203,6 +203,7 @@ public: return worst_type; } + static Vector get_main_pack_forced_export_files(); static Vector get_forced_export_files(); virtual bool fill_log_messages(RichTextLabel *p_log, Error p_err); @@ -216,9 +217,9 @@ public: virtual String get_name() const = 0; virtual Ref get_logo() const = 0; - Error export_project_files(const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func = nullptr); + Error export_project_files(bool p_main_pack, const Ref &p_preset, bool p_debug, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func = nullptr); - Error save_pack(const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files = nullptr, bool p_embed = false, int64_t *r_embedded_start = nullptr, int64_t *r_embedded_size = nullptr); + Error save_pack(bool p_main_pack, const Ref &p_preset, bool p_debug, const String &p_path, Vector *p_so_files = nullptr, bool p_embed = false, int64_t *r_embedded_start = nullptr, int64_t *r_embedded_size = nullptr); Error save_zip(const Ref &p_preset, bool p_debug, const String &p_path); virtual bool poll_export() { return false; } diff --git a/editor/export/editor_export_platform_pc.cpp b/editor/export/editor_export_platform_pc.cpp index 413db1595b3..7cd8cb9b290 100644 --- a/editor/export/editor_export_platform_pc.cpp +++ b/editor/export/editor_export_platform_pc.cpp @@ -181,7 +181,7 @@ Error EditorExportPlatformPC::export_project_data(const Ref int64_t embedded_pos; int64_t embedded_size; - Error err = save_pack(p_preset, p_debug, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); + Error err = save_pack(true, p_preset, p_debug, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); if (err == OK && p_preset->get("binary_format/embed_pck")) { if (embedded_size >= 0x100000000 && String(p_preset->get("binary_format/architecture")).contains("32")) { add_message(EXPORT_MESSAGE_ERROR, TTR("PCK Embedding"), TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); diff --git a/editor/export/project_export.cpp b/editor/export/project_export.cpp index 223900e8b30..cae814e77df 100644 --- a/editor/export/project_export.cpp +++ b/editor/export/project_export.cpp @@ -1039,6 +1039,8 @@ void ProjectExportDialog::_export_pck_zip_selected(const String &p_path) { platform->export_zip(current, export_pck_zip_debug->is_pressed(), p_path); } else if (p_path.ends_with(".pck")) { platform->export_pack(current, export_pck_zip_debug->is_pressed(), p_path); + } else { + ERR_FAIL_MSG("Path must end with .pck or .zip"); } } diff --git a/main/main.cpp b/main/main.cpp index ec66d479010..15f55fe180e 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1771,7 +1771,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph initialize_modules(MODULE_INITIALIZATION_LEVEL_CORE); register_core_extensions(); // core extensions must be registered after globals setup and before display - ResourceUID::get_singleton()->load_from_cache(); // load UUIDs from cache. + ResourceUID::get_singleton()->load_from_cache(true); // load UUIDs from cache. if (ProjectSettings::get_singleton()->has_custom_feature("dedicated_server")) { audio_driver = NULL_AUDIO_DRIVER; diff --git a/platform/android/export/export_plugin.cpp b/platform/android/export/export_plugin.cpp index 74d22bc44fd..894c13cc0b8 100644 --- a/platform/android/export/export_plugin.cpp +++ b/platform/android/export/export_plugin.cpp @@ -2643,7 +2643,7 @@ String EditorExportPlatformAndroid::get_apk_expansion_fullpath(const Ref &p_preset, bool p_debug, const String &p_path) { String fullpath = get_apk_expansion_fullpath(p_preset, p_path); - Error err = save_pack(p_preset, p_debug, fullpath); + Error err = save_pack(false, p_preset, p_debug, fullpath); return err; } @@ -3087,9 +3087,9 @@ Error EditorExportPlatformAndroid::export_project_helper(const Ref libraries; - Error err = save_pack(p_preset, p_debug, pack_path, &libraries); + Error err = save_pack(true, p_preset, p_debug, pack_path, &libraries); if (err) { // Message is supplied by the subroutine method. return err; diff --git a/platform/macos/export/export_plugin.cpp b/platform/macos/export/export_plugin.cpp index 9cc57e40665..05ae4a74c9c 100644 --- a/platform/macos/export/export_plugin.cpp +++ b/platform/macos/export/export_plugin.cpp @@ -1768,7 +1768,7 @@ Error EditorExportPlatformMacOS::export_project(const Ref &p String pack_path = tmp_app_path_name + "/Contents/Resources/" + pkg_name + ".pck"; Vector shared_objects; - err = save_pack(p_preset, p_debug, pack_path, &shared_objects); + err = save_pack(true, p_preset, p_debug, pack_path, &shared_objects); bool lib_validation = p_preset->get("codesign/entitlements/disable_library_validation"); if (!shared_objects.is_empty() && sign_enabled && ad_hoc && !lib_validation) { diff --git a/platform/web/export/export_plugin.cpp b/platform/web/export/export_plugin.cpp index 41c969b5f4a..ec7f599507e 100644 --- a/platform/web/export/export_plugin.cpp +++ b/platform/web/export/export_plugin.cpp @@ -479,7 +479,7 @@ Error EditorExportPlatformWeb::export_project(const Ref &p_p // Export pck and shared objects Vector shared_objects; String pck_path = base_path + ".pck"; - Error error = save_pack(p_preset, p_debug, pck_path, &shared_objects); + Error error = save_pack(true, p_preset, p_debug, pck_path, &shared_objects); if (error != OK) { add_message(EXPORT_MESSAGE_ERROR, TTR("Export"), vformat(TTR("Could not write file: \"%s\"."), pck_path)); return error;