GLES2 batching - prevent color baking with MODULATE or COLOR

Adding the ability to access MODULATE in the shader breaks when final_modulate is baked into vertex colors (this is a technique used to batch together different colored items). This PR prevents baking vertex colors when MODULATE is detected in the shader.

It also prevents baking when COLOR is read in canvas shaders, which could currently produce the wrong result in the shader if colors were baked. It does not prevent baking if COLOR is only written, which happens in most shaders, and will operate correctly without baking.

drivers/gles2/rasterizer_canvas_gles2.cpp

@@ -60,6 +60,7 @@ RasterizerCanvasGLES2::BatchData::BatchData() {
 	settings_colored_vertex_format_threshold = 0.0f;
 	settings_batch_buffer_num_verts = 0;
 	scissor_threshold_area = 0.0f;
+	prevent_color_baking = false;
 	diagnose_frame = false;
 	next_diagnose_tick = 10000;
 	diagnose_frame_number = 9999999999; // some high number
@@ -1687,7 +1688,9 @@ void RasterizerCanvasGLES2::flush_render_batches(Item *p_first_item, Item *p_cur
 	bdata.use_colored_vertices = false;
 
 	// only check whether to convert if there are quads (prevent divide by zero)
-	if (bdata.total_quads) {
+	// and we haven't decided to prevent color baking (due to e.g. MODULATE
+	// being used in a shader)
+	if (bdata.total_quads && !bdata.prevent_color_baking) {
 		// minus 1 to prevent single primitives (ratio 1.0) always being converted to colored..
 		// in that case it is slightly cheaper to just have the color as part of the batch
 		float ratio = (float)(bdata.total_color_changes - 1) / (float)bdata.total_quads;
@@ -2252,6 +2255,16 @@ bool RasterizerCanvasGLES2::try_join_item(Item *p_ci, RenderItemState &r_ris, bo
 	bool unshaded = r_ris.shader_cache && (r_ris.shader_cache->canvas_item.light_mode == RasterizerStorageGLES2::Shader::CanvasItem::LIGHT_MODE_UNSHADED || (blend_mode != RasterizerStorageGLES2::Shader::CanvasItem::BLEND_MODE_MIX && blend_mode != RasterizerStorageGLES2::Shader::CanvasItem::BLEND_MODE_PMALPHA));
 	bool reclip = false;
 
+	// does the shader contain BUILTINs which should break the batching?
+	if (r_ris.shader_cache && !unshaded) {
+		if (r_ris.shader_cache->canvas_item.prevent_color_baking) {
+			// we will do this same test on the shader during the rendering pass in order to set a bool not to bake vertex colors
+			// instead of saving this info as it is cheap to calculate
+			join = false;
+			r_batch_break = true;
+		}
+	}
+
 	// we are precalculating the final_modulate ahead of time because we need this for baking of final modulate into vertex colors
 	// (only in software transform mode)
 	// This maybe inefficient storing it...
@@ -2974,6 +2987,14 @@ void RasterizerCanvasGLES2::render_joined_item(const BItemJoined &p_bij, RenderI
 	bool unshaded = r_ris.shader_cache && (r_ris.shader_cache->canvas_item.light_mode == RasterizerStorageGLES2::Shader::CanvasItem::LIGHT_MODE_UNSHADED || (blend_mode != RasterizerStorageGLES2::Shader::CanvasItem::BLEND_MODE_MIX && blend_mode != RasterizerStorageGLES2::Shader::CanvasItem::BLEND_MODE_PMALPHA));
 	bool reclip = false;
 
+	// does the shader contain BUILTINs which break the batching and should prevent color baking?
+	bdata.prevent_color_baking = false;
+	if (r_ris.shader_cache && !unshaded) {
+		if (r_ris.shader_cache->canvas_item.prevent_color_baking) {
+			bdata.prevent_color_baking = true;
+		}
+	}
+
 	if (r_ris.last_blend_mode != blend_mode) {
 
 		switch (blend_mode) {

drivers/gles2/rasterizer_canvas_gles2.h

@@ -200,6 +200,10 @@ class RasterizerCanvasGLES2 : public RasterizerCanvasBaseGLES2 {
 		// to alternate batching method and add color to the vertex format.
 		int total_color_changes;
 
+		// if the shader is using MODULATE, we prevent baking so the final_modulate can
+		// be read in the shader
+		bool prevent_color_baking;
+
 		// measured in pixels, recalculated each frame
 		float scissor_threshold_area;
 

drivers/gles2/rasterizer_storage_gles2.cpp

@@ -1424,6 +1424,9 @@ void RasterizerStorageGLES2::_update_shader(Shader *p_shader) const {
 			p_shader->canvas_item.uses_screen_texture = false;
 			p_shader->canvas_item.uses_screen_uv = false;
 			p_shader->canvas_item.uses_time = false;
+			p_shader->canvas_item.uses_modulate = false;
+			p_shader->canvas_item.reads_color = false;
+			p_shader->canvas_item.prevent_color_baking = false;
 
 			shaders.actions_canvas.render_mode_values["blend_add"] = Pair<int *, int>(&p_shader->canvas_item.blend_mode, Shader::CanvasItem::BLEND_MODE_ADD);
 			shaders.actions_canvas.render_mode_values["blend_mix"] = Pair<int *, int>(&p_shader->canvas_item.blend_mode, Shader::CanvasItem::BLEND_MODE_MIX);
@@ -1438,6 +1441,9 @@ void RasterizerStorageGLES2::_update_shader(Shader *p_shader) const {
 			shaders.actions_canvas.usage_flag_pointers["SCREEN_PIXEL_SIZE"] = &p_shader->canvas_item.uses_screen_uv;
 			shaders.actions_canvas.usage_flag_pointers["SCREEN_TEXTURE"] = &p_shader->canvas_item.uses_screen_texture;
 			shaders.actions_canvas.usage_flag_pointers["TIME"] = &p_shader->canvas_item.uses_time;
+			shaders.actions_canvas.usage_flag_pointers["MODULATE"] = &p_shader->canvas_item.uses_modulate;
+
+			shaders.actions_canvas.read_flag_pointers["COLOR"] = &p_shader->canvas_item.reads_color;
 
 			actions = &shaders.actions_canvas;
 			actions->uniforms = &p_shader->uniforms;
@@ -1525,6 +1531,11 @@ void RasterizerStorageGLES2::_update_shader(Shader *p_shader) const {
 	p_shader->uses_vertex_time = gen_code.uses_vertex_time;
 	p_shader->uses_fragment_time = gen_code.uses_fragment_time;
 
+	// some logic for batching
+	if (p_shader->mode == VS::SHADER_CANVAS_ITEM) {
+		p_shader->canvas_item.prevent_color_baking = p_shader->canvas_item.uses_modulate | p_shader->canvas_item.reads_color;
+	}
+
 	p_shader->shader->set_custom_shader(p_shader->custom_code_id);
 	p_shader->shader->bind();
 

drivers/gles2/rasterizer_storage_gles2.h

@@ -450,6 +450,13 @@ public:
 			bool uses_screen_texture;
 			bool uses_screen_uv;
 			bool uses_time;
+			bool uses_modulate;
+			bool reads_color;
+
+			// this flag is specifically for batching
+			// some of the logic is thus in rasterizer_storage.cpp
+			// we could alternatively set some bitflags here and test on the fly
+			bool prevent_color_baking;
 
 		} canvas_item;
 

drivers/gles2/shader_compiler_gles2.cpp

@@ -486,6 +486,10 @@ String ShaderCompilerGLES2::_dump_node_code(SL::Node *p_node, int p_level, Gener
 				*p_actions.write_flag_pointers[var_node->name] = true;
 			}
 
+			if (!p_assigning && p_actions.read_flag_pointers.has(var_node->name)) {
+				*p_actions.read_flag_pointers[var_node->name] = true;
+			}
+
 			if (p_default_actions.usage_defines.has(var_node->name) && !used_name_defines.has(var_node->name)) {
 				String define = p_default_actions.usage_defines[var_node->name];
 				String node_name = define.substr(1, define.length());

drivers/gles2/shader_compiler_gles2.h

@@ -44,6 +44,7 @@ public:
 		Map<StringName, Pair<int *, int> > render_mode_values;
 		Map<StringName, bool *> render_mode_flags;
 		Map<StringName, bool *> usage_flag_pointers;
+		Map<StringName, bool *> read_flag_pointers;
 		Map<StringName, bool *> write_flag_pointers;
 
 		Map<StringName, ShaderLanguage::ShaderNode::Uniform> *uniforms;