Merge pull request #38121 from JFonS/add_unwrap_caching

Add caching the lightmap unwrapping on import

editor/import/resource_importer_scene.cpp

@@ -1429,11 +1429,30 @@ Error ResourceImporterScene::import(const String &p_source_file, const String &p
 		Map<Ref<ArrayMesh>, Transform> meshes;
 		_find_meshes(scene, meshes);
 
-		if (light_bake_mode == 2) {
+		String file_id = src_path.get_file();
+		String cache_file_path = base_path.plus_file(file_id + ".unwrap_cache");
+
+		Vector<unsigned char> cache_data;
+
+		if (FileAccess::exists(cache_file_path)) {
+			Error err2;
+			FileAccess *file = FileAccess::open(cache_file_path, FileAccess::READ, &err2);
+
+			if (err2) {
+				if (file)
+					memdelete(file);
+			} else {
+				int cache_size = file->get_len();
+				cache_data.resize(cache_size);
+				file->get_buffer(cache_data.ptrw(), cache_size);
+			}
+		}
 
 		float texel_size = p_options["meshes/lightmap_texel_size"];
 		texel_size = MAX(0.001, texel_size);
 
+		Map<String, unsigned int> used_unwraps;
+
 		EditorProgress progress2("gen_lightmaps", TTR("Generating Lightmaps"), meshes.size());
 		int step = 0;
 		for (Map<Ref<ArrayMesh>, Transform>::Element *E = meshes.front(); E; E = E->next()) {
@@ -1446,12 +1465,74 @@ Error ResourceImporterScene::import(const String &p_source_file, const String &p
 
 			progress2.step(TTR("Generating for Mesh: ") + name + " (" + itos(step) + "/" + itos(meshes.size()) + ")", step);
 
-				Error err2 = mesh->lightmap_unwrap(E->get(), texel_size);
+			int *ret_cache_data = (int *)cache_data.ptrw();
+			unsigned int ret_cache_size = cache_data.size();
+			bool ret_used_cache = true; // Tell the unwrapper to use the cache
+			Error err2 = mesh->lightmap_unwrap_cached(ret_cache_data, ret_cache_size, ret_used_cache, E->get(), texel_size);
+
 			if (err2 != OK) {
 				EditorNode::add_io_error("Mesh '" + name + "' failed lightmap generation. Please fix geometry.");
+			} else {
+
+				String hash = String::md5((unsigned char *)ret_cache_data);
+				used_unwraps.insert(hash, ret_cache_size);
+
+				if (!ret_used_cache) {
+					// Cache was not used, add the generated entry to the current cache
+					if (cache_data.empty()) {
+						cache_data.resize(4 + ret_cache_size);
+						int *data = (int *)cache_data.ptrw();
+						data[0] = 1;
+						memcpy(&data[1], ret_cache_data, ret_cache_size);
+					} else {
+						int current_size = cache_data.size();
+						cache_data.resize(cache_data.size() + ret_cache_size);
+						unsigned char *ptrw = cache_data.ptrw();
+						memcpy(&ptrw[current_size], ret_cache_data, ret_cache_size);
+						int *data = (int *)ptrw;
+						data[0] += 1;
+					}
+				}
 			}
 			step++;
 		}
+
+		Error err2;
+		FileAccess *file = FileAccess::open(cache_file_path, FileAccess::WRITE, &err2);
+
+		if (err2) {
+			if (file)
+				memdelete(file);
+		} else {
+
+			// Store number of entries
+			file->store_32(used_unwraps.size());
+
+			// Store cache entries
+			const int *cache = (int *)cache_data.ptr();
+			unsigned int r_idx = 1;
+			for (int i = 0; i < cache[0]; ++i) {
+				unsigned char *entry_start = (unsigned char *)&cache[r_idx];
+				String entry_hash = String::md5(entry_start);
+				if (used_unwraps.has(entry_hash)) {
+					unsigned int entry_size = used_unwraps[entry_hash];
+					file->store_buffer(entry_start, entry_size);
+				}
+
+				r_idx += 4; // hash
+				r_idx += 2; // size hint
+
+				int vertex_count = cache[r_idx];
+				r_idx += 1; // vertex count
+				r_idx += vertex_count; // vertex
+				r_idx += vertex_count * 2; // uvs
+
+				int index_count = cache[r_idx];
+				r_idx += 1; // index count
+				r_idx += index_count; // indices
+			}
+
+			file->close();
 		}
 	}
 

modules/xatlas_unwrap/register_types.cpp

@@ -32,14 +32,91 @@
 
 #include "core/error_macros.h"
 
+#include "core/crypto/crypto_core.h"
+
 #include "thirdparty/xatlas/xatlas.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 
-extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y);
+extern bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache);
 
-bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) {
+bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uvs, int **r_vertices, int *r_vertex_count, int **r_indices, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache) {
+
+	CryptoCore::MD5Context ctx;
+	ctx.start();
+
+	ctx.update((unsigned char *)&p_texel_size, sizeof(float));
+	ctx.update((unsigned char *)p_indices, sizeof(int) * p_index_count);
+	ctx.update((unsigned char *)p_vertices, sizeof(float) * p_vertex_count);
+	ctx.update((unsigned char *)p_normals, sizeof(float) * p_vertex_count);
+
+	unsigned char hash[16];
+	ctx.finish(hash);
+
+	bool cached = false;
+	unsigned int cache_idx = 0;
+
+	if (r_used_cache && r_cache_size) {
+		//Check if hash is in cache data
+
+		int *cache_data = r_cache_data;
+		int n_entries = cache_data[0];
+		unsigned int r_idx = 1;
+		for (int i = 0; i < n_entries; ++i) {
+			if (memcmp(&cache_data[r_idx], hash, 16) == 0) {
+				cached = true;
+				cache_idx = r_idx;
+				break;
+			}
+
+			r_idx += 4; // hash
+			r_idx += 2; // size hint
+
+			int vertex_count = cache_data[r_idx];
+			r_idx += 1; // vertex count
+			r_idx += vertex_count; // vertex
+			r_idx += vertex_count * 2; // uvs
+
+			int index_count = cache_data[r_idx];
+			r_idx += 1; // index count
+			r_idx += index_count; // indices
+		}
+	}
+
+	if (r_used_cache && cached) {
+		int *cache_data = r_cache_data;
+
+		// Return cache data pointer to the caller
+		r_cache_data = &cache_data[cache_idx];
+
+		cache_idx += 4;
+
+		// Load size
+		*r_size_hint_x = cache_data[cache_idx];
+		*r_size_hint_y = cache_data[cache_idx + 1];
+		cache_idx += 2;
+
+		// Load vertices
+		*r_vertex_count = cache_data[cache_idx];
+		cache_idx++;
+		*r_vertices = &cache_data[cache_idx];
+		cache_idx += *r_vertex_count;
+
+		// Load UVs
+		*r_uvs = (float *)&cache_data[cache_idx];
+		cache_idx += *r_vertex_count * 2;
+
+		// Load indices
+		*r_index_count = cache_data[cache_idx];
+		cache_idx++;
+		*r_indices = &cache_data[cache_idx];
+
+		// Return cache data size to the caller
+		r_cache_size = sizeof(int) * (4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count); // hash + size hint + vertex_count + vertices + uvs + index_count + indices
+		r_used_cache = true;
+		return true;
+	}
 
 	//set up input mesh
 	xatlas::MeshDecl input_mesh;
@@ -82,16 +159,16 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver
 
 	const xatlas::Mesh &output = atlas->meshes[0];
 
-	*r_vertex = (int *)malloc(sizeof(int) * output.vertexCount);
+	*r_vertices = (int *)malloc(sizeof(int) * output.vertexCount);
-	*r_uv = (float *)malloc(sizeof(float) * output.vertexCount * 2);
+	*r_uvs = (float *)malloc(sizeof(float) * output.vertexCount * 2);
-	*r_index = (int *)malloc(sizeof(int) * output.indexCount);
+	*r_indices = (int *)malloc(sizeof(int) * output.indexCount);
 
 	float max_x = 0;
 	float max_y = 0;
 	for (uint32_t i = 0; i < output.vertexCount; i++) {
-		(*r_vertex)[i] = output.vertexArray[i].xref;
+		(*r_vertices)[i] = output.vertexArray[i].xref;
-		(*r_uv)[i * 2 + 0] = output.vertexArray[i].uv[0] / w;
+		(*r_uvs)[i * 2 + 0] = output.vertexArray[i].uv[0] / w;
-		(*r_uv)[i * 2 + 1] = output.vertexArray[i].uv[1] / h;
+		(*r_uvs)[i * 2 + 1] = output.vertexArray[i].uv[1] / h;
 		max_x = MAX(max_x, output.vertexArray[i].uv[0]);
 		max_y = MAX(max_y, output.vertexArray[i].uv[1]);
 	}
@@ -100,13 +177,54 @@ bool xatlas_mesh_lightmap_unwrap_callback(float p_texel_size, const float *p_ver
 	*r_vertex_count = output.vertexCount;
 
 	for (uint32_t i = 0; i < output.indexCount; i++) {
-		(*r_index)[i] = output.indexArray[i];
+		(*r_indices)[i] = output.indexArray[i];
 	}
 
 	*r_index_count = output.indexCount;
 
 	xatlas::Destroy(atlas);
-	printf("Done\n");
+
+	if (r_used_cache) {
+		unsigned int new_cache_size = 4 + 2 + 1 + *r_vertex_count + (*r_vertex_count * 2) + 1 + *r_index_count; // hash + size hint + vertex_count + vertices + uvs + index_count + indices
+		new_cache_size *= sizeof(int);
+		int *new_cache_data = (int *)memalloc(new_cache_size);
+		unsigned int new_cache_idx = 0;
+
+		// hash
+		memcpy(&new_cache_data[new_cache_idx], hash, 16);
+		new_cache_idx += 4;
+
+		// size hint
+		new_cache_data[new_cache_idx] = *r_size_hint_x;
+		new_cache_data[new_cache_idx + 1] = *r_size_hint_y;
+		new_cache_idx += 2;
+
+		// vertex count
+		new_cache_data[new_cache_idx] = *r_vertex_count;
+		new_cache_idx++;
+
+		// vertices
+		memcpy(&new_cache_data[new_cache_idx], *r_vertices, sizeof(int) * *r_vertex_count);
+		new_cache_idx += *r_vertex_count;
+
+		// uvs
+		memcpy(&new_cache_data[new_cache_idx], *r_uvs, sizeof(float) * *r_vertex_count * 2);
+		new_cache_idx += *r_vertex_count * 2;
+
+		// index count
+		new_cache_data[new_cache_idx] = *r_index_count;
+		new_cache_idx++;
+
+		// indices
+		memcpy(&new_cache_data[new_cache_idx], *r_indices, sizeof(int) * *r_index_count);
+		new_cache_idx += *r_index_count;
+
+		// Return cache data to the caller
+		r_cache_data = new_cache_data;
+		r_cache_size = new_cache_size;
+		r_used_cache = false;
+	}
+
 	return true;
 }
 

scene/resources/mesh.cpp

@@ -1359,7 +1359,7 @@ void ArrayMesh::regen_normalmaps() {
 }
 
 //dirty hack
-bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, const int *p_face_materials, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y) = nullptr;
+bool (*array_mesh_lightmap_unwrap_callback)(float p_texel_size, const float *p_vertices, const float *p_normals, int p_vertex_count, const int *p_indices, int p_index_count, float **r_uv, int **r_vertex, int *r_vertex_count, int **r_index, int *r_index_count, int *r_size_hint_x, int *r_size_hint_y, int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache);
 
 struct ArrayMeshLightmapSurface {
 
@@ -1370,6 +1370,13 @@ struct ArrayMeshLightmapSurface {
 };
 
 Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texel_size) {
+	int *cache_data = nullptr;
+	unsigned int cache_size = 0;
+	bool use_cache = false; // Don't use cache
+	return lightmap_unwrap_cached(cache_data, cache_size, use_cache, p_base_transform, p_texel_size);
+}
+
+Error ArrayMesh::lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform, float p_texel_size) {
 
 	ERR_FAIL_COND_V(!array_mesh_lightmap_unwrap_callback, ERR_UNCONFIGURED);
 	ERR_FAIL_COND_V_MSG(blend_shapes.size() != 0, ERR_UNAVAILABLE, "Can't unwrap mesh with blend shapes.");
@@ -1377,11 +1384,18 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 	Vector<float> vertices;
 	Vector<float> normals;
 	Vector<int> indices;
-	Vector<int> face_materials;
 	Vector<float> uv;
-	Vector<Pair<int, int>> uv_index;
+	Vector<Pair<int, int>> uv_indices;
+
+	Vector<ArrayMeshLightmapSurface> lightmap_surfaces;
+
+	// Keep only the scale
+	Transform transform = p_base_transform;
+	transform.origin = Vector3();
+	transform.looking_at(Vector3(1, 0, 0), Vector3(0, 1, 0));
+
+	Basis normal_basis = transform.basis.inverse().transposed();
 
-	Vector<ArrayMeshLightmapSurface> surfaces;
 	for (int i = 0; i < get_surface_count(); i++) {
 		ArrayMeshLightmapSurface s;
 		s.primitive = surface_get_primitive_type(i);
@@ -1405,12 +1419,12 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 
 		vertices.resize((vertex_ofs + vc) * 3);
 		normals.resize((vertex_ofs + vc) * 3);
-		uv_index.resize(vertex_ofs + vc);
+		uv_indices.resize(vertex_ofs + vc);
 
 		for (int j = 0; j < vc; j++) {
 
-			Vector3 v = p_base_transform.xform(r[j]);
+			Vector3 v = transform.xform(r[j]);
-			Vector3 n = p_base_transform.basis.xform(rn[j]).normalized();
+			Vector3 n = normal_basis.xform(rn[j]).normalized();
 
 			vertices.write[(j + vertex_ofs) * 3 + 0] = v.x;
 			vertices.write[(j + vertex_ofs) * 3 + 1] = v.y;
@@ -1418,7 +1432,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 			normals.write[(j + vertex_ofs) * 3 + 0] = n.x;
 			normals.write[(j + vertex_ofs) * 3 + 1] = n.y;
 			normals.write[(j + vertex_ofs) * 3 + 2] = n.z;
-			uv_index.write[j + vertex_ofs] = Pair<int, int>(i, j);
+			uv_indices.write[j + vertex_ofs] = Pair<int, int>(i, j);
 		}
 
 		Vector<int> rindices = arrays[Mesh::ARRAY_INDEX];
@@ -1433,7 +1447,6 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 				indices.push_back(vertex_ofs + j * 3 + 0);
 				indices.push_back(vertex_ofs + j * 3 + 1);
 				indices.push_back(vertex_ofs + j * 3 + 2);
-				face_materials.push_back(i);
 			}
 
 		} else {
@@ -1445,11 +1458,10 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 				indices.push_back(vertex_ofs + ri[j * 3 + 0]);
 				indices.push_back(vertex_ofs + ri[j * 3 + 1]);
 				indices.push_back(vertex_ofs + ri[j * 3 + 2]);
-				face_materials.push_back(i);
 			}
 		}
 
-		surfaces.push_back(s);
+		lightmap_surfaces.push_back(s);
 	}
 
 	//unwrap
@@ -1462,7 +1474,7 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 	int size_x;
 	int size_y;
 
-	bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), face_materials.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y);
+	bool ok = array_mesh_lightmap_unwrap_callback(p_texel_size, vertices.ptr(), normals.ptr(), vertices.size() / 3, indices.ptr(), indices.size(), &gen_uvs, &gen_vertices, &gen_vertex_count, &gen_indices, &gen_index_count, &size_x, &size_y, r_cache_data, r_cache_size, r_used_cache);
 
 	if (!ok) {
 		return ERR_CANT_CREATE;
@@ -1474,11 +1486,11 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 	//create surfacetools for each surface..
 	Vector<Ref<SurfaceTool>> surfaces_tools;
 
-	for (int i = 0; i < surfaces.size(); i++) {
+	for (int i = 0; i < lightmap_surfaces.size(); i++) {
 		Ref<SurfaceTool> st;
 		st.instance();
 		st->begin(Mesh::PRIMITIVE_TRIANGLES);
-		st->set_material(surfaces[i].material);
+		st->set_material(lightmap_surfaces[i].material);
 		surfaces_tools.push_back(st); //stay there
 	}
 
@@ -1486,37 +1498,37 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 	//go through all indices
 	for (int i = 0; i < gen_index_count; i += 3) {
 
-		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_index.size(), ERR_BUG);
+		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 0]], uv_indices.size(), ERR_BUG);
-		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_index.size(), ERR_BUG);
+		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 1]], uv_indices.size(), ERR_BUG);
-		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_index.size(), ERR_BUG);
+		ERR_FAIL_INDEX_V(gen_vertices[gen_indices[i + 2]], uv_indices.size(), ERR_BUG);
 
-		ERR_FAIL_COND_V(uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 1]]].first || uv_index[gen_vertices[gen_indices[i + 0]]].first != uv_index[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG);
+		ERR_FAIL_COND_V(uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 1]]].first || uv_indices[gen_vertices[gen_indices[i + 0]]].first != uv_indices[gen_vertices[gen_indices[i + 2]]].first, ERR_BUG);
 
-		int surface = uv_index[gen_vertices[gen_indices[i + 0]]].first;
+		int surface = uv_indices[gen_vertices[gen_indices[i + 0]]].first;
 
 		for (int j = 0; j < 3; j++) {
 
-			SurfaceTool::Vertex v = surfaces[surface].vertices[uv_index[gen_vertices[gen_indices[i + j]]].second];
+			SurfaceTool::Vertex v = lightmap_surfaces[surface].vertices[uv_indices[gen_vertices[gen_indices[i + j]]].second];
 
-			if (surfaces[surface].format & ARRAY_FORMAT_COLOR) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_COLOR) {
 				surfaces_tools.write[surface]->add_color(v.color);
 			}
-			if (surfaces[surface].format & ARRAY_FORMAT_TEX_UV) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TEX_UV) {
 				surfaces_tools.write[surface]->add_uv(v.uv);
 			}
-			if (surfaces[surface].format & ARRAY_FORMAT_NORMAL) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_NORMAL) {
 				surfaces_tools.write[surface]->add_normal(v.normal);
 			}
-			if (surfaces[surface].format & ARRAY_FORMAT_TANGENT) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_TANGENT) {
 				Plane t;
 				t.normal = v.tangent;
 				t.d = v.binormal.dot(v.normal.cross(v.tangent)) < 0 ? -1 : 1;
 				surfaces_tools.write[surface]->add_tangent(t);
 			}
-			if (surfaces[surface].format & ARRAY_FORMAT_BONES) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_BONES) {
 				surfaces_tools.write[surface]->add_bones(v.bones);
 			}
-			if (surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) {
+			if (lightmap_surfaces[surface].format & ARRAY_FORMAT_WEIGHTS) {
 				surfaces_tools.write[surface]->add_weights(v.weights);
 			}
 
@@ -1527,20 +1539,22 @@ Error ArrayMesh::lightmap_unwrap(const Transform &p_base_transform, float p_texe
 		}
 	}
 
-	//free stuff
-	::free(gen_vertices);
-	::free(gen_indices);
-	::free(gen_uvs);
-
 	//generate surfaces
 
 	for (int i = 0; i < surfaces_tools.size(); i++) {
 		surfaces_tools.write[i]->index();
-		surfaces_tools.write[i]->commit(Ref<ArrayMesh>((ArrayMesh *)this), surfaces[i].format);
+		surfaces_tools.write[i]->commit(Ref<ArrayMesh>((ArrayMesh *)this), lightmap_surfaces[i].format);
 	}
 
 	set_lightmap_size_hint(Size2(size_x, size_y));
 
+	if (!r_used_cache) {
+		//free stuff
+		::free(gen_vertices);
+		::free(gen_indices);
+		::free(gen_uvs);
+	}
+
 	return OK;
 }
 

scene/resources/mesh.h

@@ -238,6 +238,7 @@ public:
 	void regen_normalmaps();
 
 	Error lightmap_unwrap(const Transform &p_base_transform = Transform(), float p_texel_size = 0.05);
+	Error lightmap_unwrap_cached(int *&r_cache_data, unsigned int &r_cache_size, bool &r_used_cache, const Transform &p_base_transform = Transform(), float p_texel_size = 0.05);
 
 	virtual void reload_from_file();