Add embedded PCK option to PC platforms

The basic point is as in 2.1 (appending the PCK into the executable), but this implementation also patches a dedicated section in the ELF/PE executable so that it matches the appended data perfectly.

The usage of integer types is simplified in existing code; namely, using plain `int` for small quantities.

core/io/file_access_pack.cpp

@@ -144,7 +144,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path) {
 	uint32_t magic = f->get_32();
 
 	if (magic != 0x43504447) {
-		//maybe at he end.... self contained exe
+		//maybe at the end.... self contained exe
 		f->seek_end();
 		f->seek(f->get_position() - 4);
 		magic = f->get_32();

core/project_settings.cpp

@@ -343,6 +343,9 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 		return err;
 	}
 
+	String exec_path = OS::get_singleton()->get_executable_path();
+
+	if (exec_path != "") {
 		// Attempt with exec_name.pck
 		// (This is the usual case when distributing a Godot game.)
 
@@ -351,9 +354,6 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 		// We need to test both possibilities as extensions for Linux binaries are optional
 		// (so both 'mygame.bin' and 'mygame' should be able to find 'mygame.pck').
 
-	String exec_path = OS::get_singleton()->get_executable_path();
-
-	if (exec_path != "") {
 		bool found = false;
 
 		String exec_dir = exec_path.get_base_dir();
@@ -375,6 +375,14 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 			}
 		}
 
+		// Attempt with PCK bundled into executable
+
+		if (!found) {
+			if (_load_resource_pack(exec_path)) {
+				found = true;
+			}
+		}
+
 		// If we opened our package, try and load our project
 		if (found) {
 			Error err = _load_settings_text_or_binary("res://project.godot", "res://project.binary");

editor/editor_export.cpp

@@ -901,7 +901,7 @@ Error EditorExportPlatform::_add_shared_object(void *p_userdata, const SharedObj
 	return OK;
 }
 
-Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, const String &p_path, Vector<SharedObject> *p_so_files) {
+Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, const String &p_path, Vector<SharedObject> *p_so_files, bool p_embed, int64_t *r_embedded_start, int64_t *r_embedded_size) {
 
 	EditorProgress ep("savepack", TTR("Packing"), 102, true);
 
@@ -923,9 +923,34 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 
 	pd.file_ofs.sort(); //do sort, so we can do binary search later
 
-	FileAccess *f = FileAccess::open(p_path, FileAccess::WRITE);
+	FileAccess *f;
+	int64_t embed_pos = 0;
+	if (!p_embed) {
+		// Regular output to separate PCK file
+		f = FileAccess::open(p_path, FileAccess::WRITE);
 		ERR_FAIL_COND_V(!f, ERR_CANT_CREATE);
-	f->store_32(0x43504447); //GDPK
+	} else {
+		// Append to executable
+		f = FileAccess::open(p_path, FileAccess::READ_WRITE);
+		ERR_FAIL_COND_V(!f, ERR_FILE_CANT_OPEN);
+
+		f->seek_end();
+		embed_pos = f->get_position();
+
+		if (r_embedded_start) {
+			*r_embedded_start = embed_pos;
+		}
+
+		// Ensure embedded PCK starts at a 64-bit multiple
+		int pad = f->get_position() % 8;
+		for (int i = 0; i < pad; i++) {
+			f->store_8(0);
+		}
+	}
+
+	int64_t pck_start_pos = f->get_position();
+
+	f->store_32(0x43504447); //GDPC
 	f->store_32(1); //pack version
 	f->store_32(VERSION_MAJOR);
 	f->store_32(VERSION_MINOR);
@@ -937,29 +962,29 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 
 	f->store_32(pd.file_ofs.size()); //amount of files
 
-	size_t header_size = f->get_position();
+	int64_t header_size = f->get_position();
 
 	//precalculate header size
 
 	for (int i = 0; i < pd.file_ofs.size(); i++) {
 		header_size += 4; // size of path string (32 bits is enough)
-		uint32_t string_len = pd.file_ofs[i].path_utf8.length();
+		int string_len = pd.file_ofs[i].path_utf8.length();
 		header_size += string_len + _get_pad(4, string_len); ///size of path string
 		header_size += 8; // offset to file _with_ header size included
 		header_size += 8; // size of file
 		header_size += 16; // md5
 	}
 
-	size_t header_padding = _get_pad(PCK_PADDING, header_size);
+	int header_padding = _get_pad(PCK_PADDING, header_size);
 
 	for (int i = 0; i < pd.file_ofs.size(); i++) {
 
-		uint32_t string_len = pd.file_ofs[i].path_utf8.length();
+		int string_len = pd.file_ofs[i].path_utf8.length();
-		uint32_t pad = _get_pad(4, string_len);
+		int pad = _get_pad(4, string_len);
-		;
+
 		f->store_32(string_len + pad);
 		f->store_buffer((const uint8_t *)pd.file_ofs[i].path_utf8.get_data(), string_len);
-		for (uint32_t j = 0; j < pad; j++) {
+		for (int j = 0; j < pad; j++) {
 			f->store_8(0);
 		}
 
@@ -968,7 +993,7 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 		f->store_buffer(pd.file_ofs[i].md5.ptr(), 16); //also save md5 for file
 	}
 
-	for (uint32_t j = 0; j < header_padding; j++) {
+	for (int i = 0; i < header_padding; i++) {
 		f->store_8(0);
 	}
 
@@ -994,7 +1019,23 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 
 	memdelete(ftmp);
 
-	f->store_32(0x43504447); //GDPK
+	if (p_embed) {
+		// Ensure embedded data ends at a 64-bit multiple
+		int64_t embed_end = f->get_position() - embed_pos + 12;
+		int pad = embed_end % 8;
+		for (int i = 0; i < pad; i++) {
+			f->store_8(0);
+		}
+
+		int64_t pck_size = f->get_position() - pck_start_pos;
+		f->store_64(pck_size);
+		f->store_32(0x43504447); //GDPC
+
+		if (r_embedded_size) {
+			*r_embedded_size = f->get_position() - embed_pos;
+		}
+	}
+
 	memdelete(f);
 
 	return OK;
@@ -1401,6 +1442,7 @@ void EditorExportPlatformPC::get_export_options(List<ExportOption> *r_options) {
 	r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "texture_format/etc2"), false));
 	r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "texture_format/no_bptc_fallbacks"), true));
 	r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "binary_format/64_bits"), true));
+	r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "binary_format/embed_pck"), false));
 	r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "custom_template/release", PROPERTY_HINT_GLOBAL_FILE), ""));
 	r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "custom_template/debug", PROPERTY_HINT_GLOBAL_FILE), ""));
 }
@@ -1518,12 +1560,33 @@ Error EditorExportPlatformPC::export_project(const Ref<EditorExportPreset> &p_pr
 
 	DirAccess *da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
 	Error err = da->copy(template_path, p_path, get_chmod_flags());
+	memdelete(da);
+
 	if (err == OK) {
-		String pck_path = p_path.get_basename() + ".pck";
+		String pck_path;
+		if (p_preset->get("binary_format/embed_pck")) {
+			pck_path = p_path;
+		} else {
+			pck_path = p_path.get_basename() + ".pck";
+		}
 
 		Vector<SharedObject> so_files;
 
-		err = save_pack(p_preset, pck_path, &so_files);
+		int64_t embedded_pos;
+		int64_t embedded_size;
+		err = save_pack(p_preset, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size);
+		if (err == OK && p_preset->get("binary_format/embed_pck")) {
+
+			if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) {
+				EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB."));
+				return ERR_UNAVAILABLE;
+			}
+
+			FixUpEmbeddedPckFunc fixup_func = get_fixup_embedded_pck_func();
+			if (fixup_func) {
+				err = fixup_func(p_path, embedded_pos, embedded_size);
+			}
+		}
 
 		if (err == OK && !so_files.empty()) {
 			//if shared object files, copy them
@@ -1531,10 +1594,10 @@ Error EditorExportPlatformPC::export_project(const Ref<EditorExportPreset> &p_pr
 			for (int i = 0; i < so_files.size() && err == OK; i++) {
 				err = da->copy(so_files[i].path, p_path.get_base_dir().plus_file(so_files[i].path.get_file()));
 			}
+			memdelete(da);
 		}
 	}
 
-	memdelete(da);
 	return err;
 }
 
@@ -1605,9 +1668,20 @@ void EditorExportPlatformPC::set_chmod_flags(int p_flags) {
 	chmod_flags = p_flags;
 }
 
+EditorExportPlatformPC::FixUpEmbeddedPckFunc EditorExportPlatformPC::get_fixup_embedded_pck_func() const {
+
+	return fixup_embedded_pck_func;
+}
+
+void EditorExportPlatformPC::set_fixup_embedded_pck_func(FixUpEmbeddedPckFunc p_fixup_embedded_pck_func) {
+
+	fixup_embedded_pck_func = p_fixup_embedded_pck_func;
+}
+
 EditorExportPlatformPC::EditorExportPlatformPC() {
 
 	chmod_flags = -1;
+	fixup_embedded_pck_func = NULL;
 }
 
 ///////////////////////

editor/editor_export.h

@@ -240,7 +240,7 @@ public:
 
 	Error export_project_files(const Ref<EditorExportPreset> &p_preset, EditorExportSaveFunction p_func, void *p_udata, EditorExportSaveSharedObject p_so_func = NULL);
 
-	Error save_pack(const Ref<EditorExportPreset> &p_preset, const String &p_path, Vector<SharedObject> *p_so_files = NULL);
+	Error save_pack(const Ref<EditorExportPreset> &p_preset, const String &p_path, Vector<SharedObject> *p_so_files = NULL, bool p_embed = false, int64_t *r_embedded_start = NULL, int64_t *r_embedded_size = NULL);
 	Error save_zip(const Ref<EditorExportPreset> &p_preset, const String &p_path);
 
 	virtual bool poll_devices() { return false; }
@@ -391,6 +391,10 @@ class EditorExportPlatformPC : public EditorExportPlatform {
 
 	GDCLASS(EditorExportPlatformPC, EditorExportPlatform);
 
+public:
+	typedef Error (*FixUpEmbeddedPckFunc)(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size);
+
+private:
 	Ref<ImageTexture> logo;
 	String name;
 	String os_name;
@@ -405,6 +409,8 @@ class EditorExportPlatformPC : public EditorExportPlatform {
 
 	int chmod_flags;
 
+	FixUpEmbeddedPckFunc fixup_embedded_pck_func;
+
 public:
 	virtual void get_preset_features(const Ref<EditorExportPreset> &p_preset, List<String> *r_features);
 
@@ -436,6 +442,9 @@ public:
 	int get_chmod_flags() const;
 	void set_chmod_flags(int p_flags);
 
+	FixUpEmbeddedPckFunc get_fixup_embedded_pck_func() const;
+	void set_fixup_embedded_pck_func(FixUpEmbeddedPckFunc p_fixup_embedded_pck_func);
+
 	EditorExportPlatformPC();
 };
 

platform/windows/export/export.cpp

@@ -34,6 +34,8 @@
 #include "editor/editor_settings.h"
 #include "platform/windows/logo.gen.h"
 
+static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size);
+
 class EditorExportPlatformWindows : public EditorExportPlatformPC {
 
 public:
@@ -172,6 +174,80 @@ void register_windows_exporter() {
 	platform->set_release_64("windows_64_release.exe");
 	platform->set_debug_64("windows_64_debug.exe");
 	platform->set_os_name("Windows");
+	platform->set_fixup_embedded_pck_func(&fixup_embedded_pck);
 
 	EditorExport::get_singleton()->add_export_platform(platform);
 }
+
+static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size) {
+
+	// Patch the header of the "pck" section in the PE file so that it corresponds to the embedded data
+
+	FileAccess *f = FileAccess::open(p_path, FileAccess::READ_WRITE);
+	if (!f) {
+		return ERR_CANT_OPEN;
+	}
+
+	// Jump to the PE header and check the magic number
+	{
+		f->seek(0x3c);
+		uint32_t pe_pos = f->get_32();
+
+		f->seek(pe_pos);
+		uint32_t magic = f->get_32();
+		if (magic != 0x00004550) {
+			f->close();
+			return ERR_FILE_CORRUPT;
+		}
+	}
+
+	// Process header
+
+	int num_sections;
+	{
+		int64_t header_pos = f->get_position();
+
+		f->seek(header_pos + 2);
+		num_sections = f->get_16();
+		f->seek(header_pos + 16);
+		uint16_t opt_header_size = f->get_16();
+
+		// Skip rest of header + optional header to go to the section headers
+		f->seek(f->get_position() + 2 + opt_header_size);
+	}
+
+	// Search for the "pck" section
+
+	int64_t section_table_pos = f->get_position();
+
+	bool found = false;
+	for (int i = 0; i < num_sections; ++i) {
+
+		int64_t section_header_pos = section_table_pos + i * 40;
+		f->seek(section_header_pos);
+
+		uint8_t section_name[9];
+		f->get_buffer(section_name, 8);
+		section_name[8] = '\0';
+
+		if (strcmp((char *)section_name, "pck") == 0) {
+			// "pck" section found, let's patch!
+
+			// Set virtual size to a little to avoid it taking memory (zero would give issues)
+			f->seek(section_header_pos + 8);
+			f->store_32(8);
+
+			f->seek(section_header_pos + 16);
+			f->store_32(p_embedded_size);
+			f->seek(section_header_pos + 20);
+			f->store_32(p_embedded_start);
+
+			found = true;
+			break;
+		}
+	}
+
+	f->close();
+
+	return found ? OK : ERR_FILE_CORRUPT;
+}

platform/windows/godot_windows.cpp

@@ -34,6 +34,17 @@
 #include <locale.h>
 #include <stdio.h>
 
+// For export templates, add a section; the exporter will patch it to enclose
+// the data appended to the executable (bundled PCK)
+#ifndef TOOLS_ENABLED
+#if defined _MSC_VER
+#pragma section("pck", read)
+__declspec(allocate("pck")) static char dummy[8] = { 0 };
+#elif defined __GNUC__
+static const char dummy[8] __attribute__((section("pck"), used)) = { 0 };
+#endif
+#endif
+
 PCHAR *
 CommandLineToArgvA(
 		PCHAR CmdLine,

platform/x11/detect.py

@@ -325,6 +325,9 @@ def configure(env):
     if env["execinfo"]:
         env.Append(LIBS=['execinfo'])
         
+    if not env['tools']:
+        env.Append(LINKFLAGS=['-T', 'platform/x11/pck_embed.ld'])
+
     ## Cross-compilation
 
     if (is64 and env["bits"] == "32"):

platform/x11/export/export.cpp

@@ -30,10 +30,13 @@
 
 #include "export.h"
 
+#include "core/os/file_access.h"
 #include "editor/editor_export.h"
 #include "platform/x11/logo.gen.h"
 #include "scene/resources/texture.h"
 
+static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size);
+
 void register_x11_exporter() {
 
 	Ref<EditorExportPlatformPC> platform;
@@ -53,6 +56,115 @@ void register_x11_exporter() {
 	platform->set_debug_64("linux_x11_64_debug");
 	platform->set_os_name("X11");
 	platform->set_chmod_flags(0755);
+	platform->set_fixup_embedded_pck_func(&fixup_embedded_pck);
 
 	EditorExport::get_singleton()->add_export_platform(platform);
 }
+
+static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size) {
+
+	// Patch the header of the "pck" section in the ELF file so that it corresponds to the embedded data
+
+	FileAccess *f = FileAccess::open(p_path, FileAccess::READ_WRITE);
+	if (!f) {
+		return ERR_CANT_OPEN;
+	}
+
+	// Read and check ELF magic number
+	{
+		uint32_t magic = f->get_32();
+		if (magic != 0x464c457f) { // 0x7F + "ELF"
+			f->close();
+			return ERR_FILE_CORRUPT;
+		}
+	}
+
+	// Read program architecture bits from class field
+
+	int bits = f->get_8() * 32;
+
+	if (bits == 32 && p_embedded_size >= 0x100000000) {
+		f->close();
+		ERR_EXPLAIN("32-bit executables cannot have embedded data >= 4 GiB");
+		ERR_FAIL_V(ERR_INVALID_DATA);
+	}
+
+	// Get info about the section header table
+
+	int64_t section_table_pos;
+	int64_t section_header_size;
+	if (bits == 32) {
+		section_header_size = 40;
+		f->seek(0x20);
+		section_table_pos = f->get_32();
+		f->seek(0x30);
+	} else { // 64
+		section_header_size = 64;
+		f->seek(0x28);
+		section_table_pos = f->get_64();
+		f->seek(0x3c);
+	}
+	int num_sections = f->get_16();
+	int string_section_idx = f->get_16();
+
+	// Load the strings table
+	uint8_t *strings;
+	{
+		// Jump to the strings section header
+		f->seek(section_table_pos + string_section_idx * section_header_size);
+
+		// Read strings data size and offset
+		int64_t string_data_pos;
+		int64_t string_data_size;
+		if (bits == 32) {
+			f->seek(f->get_position() + 0x10);
+			string_data_pos = f->get_32();
+			string_data_size = f->get_32();
+		} else { // 64
+			f->seek(f->get_position() + 0x18);
+			string_data_pos = f->get_64();
+			string_data_size = f->get_64();
+		}
+
+		// Read strings data
+		f->seek(string_data_pos);
+		strings = (uint8_t *)memalloc(string_data_size);
+		if (!strings) {
+			f->close();
+			return ERR_OUT_OF_MEMORY;
+		}
+		f->get_buffer(strings, string_data_size);
+	}
+
+	// Search for the "pck" section
+
+	bool found = false;
+	for (int i = 0; i < num_sections; ++i) {
+
+		int64_t section_header_pos = section_table_pos + i * section_header_size;
+		f->seek(section_header_pos);
+
+		uint32_t name_offset = f->get_32();
+		if (strcmp((char *)strings + name_offset, "pck") == 0) {
+			// "pck" section found, let's patch!
+
+			if (bits == 32) {
+				f->seek(section_header_pos + 0x10);
+				f->store_32(p_embedded_start);
+				f->store_32(p_embedded_size);
+			} else { // 64
+				f->seek(section_header_pos + 0x18);
+				f->store_64(p_embedded_start);
+				f->store_64(p_embedded_size);
+			}
+
+			found = true;
+			break;
+		}
+	}
+
+	memfree(strings);
+	f->close();
+
+	return found ? OK : ERR_FILE_CORRUPT;
+}

platform/x11/pck_embed.ld

@@ -0,0 +1,10 @@
+SECTIONS
+{
+	/* Add a zero-sized section; the exporter will patch it to enclose the data appended to the executable (embedded PCK) */
+	pck 0 (NOLOAD) :
+	{
+		/* Just some content to avoid the linker discarding the section */
+		. = ALIGN(8);
+	}
+}
+INSERT AFTER .rodata;