-Fix all shadow and culling related issues, fixes #9330

drivers/gles3/rasterizer_scene_gles3.cpp

@@ -1088,11 +1088,12 @@ void RasterizerSceneGLES3::gi_probe_instance_set_bounds(RID p_probe, const Vecto
 
 bool RasterizerSceneGLES3::_setup_material(RasterizerStorageGLES3::Material *p_material, bool p_alpha_pass) {
 
+	/* this is handled outside
 	if (p_material->shader->spatial.cull_mode == RasterizerStorageGLES3::Shader::Spatial::CULL_MODE_DISABLED) {
 		glDisable(GL_CULL_FACE);
 	} else {
 		glEnable(GL_CULL_FACE);
-	}
+	} */
 
 	if (state.current_line_width != p_material->line_width) {
 		//glLineWidth(MAX(p_material->line_width,1.0));
@@ -1857,12 +1858,21 @@ void RasterizerSceneGLES3::_setup_light(RenderList::Element *e, const Transform
 	}
 }
 
-void RasterizerSceneGLES3::_set_cull(bool p_front, bool p_reverse_cull) {
+void RasterizerSceneGLES3::_set_cull(bool p_front, bool p_disabled, bool p_reverse_cull) {
 
 	bool front = p_front;
 	if (p_reverse_cull)
 		front = !front;
 
+	if (p_disabled != state.cull_disabled) {
+		if (p_disabled)
+			glDisable(GL_CULL_FACE);
+		else
+			glEnable(GL_CULL_FACE);
+
+		state.cull_disabled = p_disabled;
+	}
+
 	if (front != state.cull_front) {
 
 		glCullFace(front ? GL_FRONT : GL_BACK);
@@ -1900,7 +1910,9 @@ void RasterizerSceneGLES3::_render_list(RenderList::Element **p_elements, int p_
 	}
 
 	state.cull_front = false;
+	state.cull_disabled = false;
 	glCullFace(GL_BACK);
+	glEnable(GL_CULL_FACE);
 
 	state.current_depth_test = true;
 	glEnable(GL_DEPTH_TEST);
@@ -2101,7 +2113,7 @@ void RasterizerSceneGLES3::_render_list(RenderList::Element **p_elements, int p_
 			storage->info.render.surface_switch_count++;
 		}
 
-		_set_cull(e->sort_key & RenderList::SORT_KEY_MIRROR_FLAG, p_reverse_cull);
+		_set_cull(e->sort_key & RenderList::SORT_KEY_MIRROR_FLAG, e->sort_key & RenderList::SORT_KEY_CULL_DISABLED_FLAG, p_reverse_cull);
 
 		state.scene_shader.set_uniform(SceneShaderGLES3::NORMAL_MULT, e->instance->mirror ? -1.0 : 1.0);
 		state.scene_shader.set_uniform(SceneShaderGLES3::WORLD_TRANSFORM, e->instance->transform);
@@ -2188,8 +2200,12 @@ void RasterizerSceneGLES3::_add_geometry_with_material(RasterizerStorageGLES3::G
 	bool shadow = false;
 
 	bool mirror = p_instance->mirror;
+	bool no_cull = false;
 
-	if (p_material->shader->spatial.cull_mode == RasterizerStorageGLES3::Shader::Spatial::CULL_MODE_FRONT) {
+	if (p_material->shader->spatial.cull_mode == RasterizerStorageGLES3::Shader::Spatial::CULL_MODE_DISABLED) {
+		no_cull = true;
+		mirror = false;
+	} else if (p_material->shader->spatial.cull_mode == RasterizerStorageGLES3::Shader::Spatial::CULL_MODE_FRONT) {
 		mirror = !mirror;
 	}
 
@@ -2208,11 +2224,14 @@ void RasterizerSceneGLES3::_add_geometry_with_material(RasterizerStorageGLES3::G
 
 		if (!p_material->shader->spatial.uses_alpha_scissor && !p_material->shader->spatial.writes_modelview_or_projection && !p_material->shader->spatial.uses_vertex && !p_material->shader->spatial.uses_discard && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) {
 			//shader does not use discard and does not write a vertex position, use generic material
-			if (p_instance->cast_shadows == VS::SHADOW_CASTING_SETTING_DOUBLE_SIDED)
+			if (p_instance->cast_shadows == VS::SHADOW_CASTING_SETTING_DOUBLE_SIDED) {
 				p_material = storage->material_owner.getptr(default_material_twosided);
-			else
+				no_cull = true;
+				mirror = false;
+			} else {
 				p_material = storage->material_owner.getptr(default_material);
 			}
+		}
 
 		has_alpha = false;
 	}
@@ -2275,6 +2294,10 @@ void RasterizerSceneGLES3::_add_geometry_with_material(RasterizerStorageGLES3::G
 		e->sort_key |= RenderList::SORT_KEY_MIRROR_FLAG;
 	}
 
+	if (no_cull) {
+		e->sort_key |= RenderList::SORT_KEY_CULL_DISABLED_FLAG;
+	}
+
 	//e->light_type=0xFF; // no lights!
 
 	if (shadow || p_material->shader->spatial.unshaded || state.debug_draw == VS::VIEWPORT_DEBUG_DRAW_UNSHADED) {
@@ -4535,6 +4558,9 @@ void RasterizerSceneGLES3::render_shadow(RID p_light, RID p_shadow_atlas, int p_
 
 	state.scene_shader.set_conditional(SceneShaderGLES3::RENDER_DEPTH, true);
 
+	if (light->reverse_cull) {
+		flip_facing = !flip_facing;
+	}
 	_render_list(render_list.elements, render_list.element_count, light_transform, light_projection, 0, flip_facing, false, true, false, false);
 
 	state.scene_shader.set_conditional(SceneShaderGLES3::RENDER_DEPTH, false);

drivers/gles3/rasterizer_scene_gles3.h

@@ -187,6 +187,7 @@ public:
 		int reflection_probe_count;
 
 		bool cull_front;
+		bool cull_disabled;
 		bool used_sss;
 		bool used_screen_texture;
 		bool using_contact_shadows;
@@ -654,6 +655,7 @@ public:
 			SORT_KEY_MATERIAL_INDEX_SHIFT = 40,
 			SORT_KEY_GEOMETRY_INDEX_SHIFT = 20,
 			SORT_KEY_GEOMETRY_TYPE_SHIFT = 15,
+			SORT_KEY_CULL_DISABLED_FLAG = 4,
 			SORT_KEY_SKELETON_FLAG = 2,
 			SORT_KEY_MIRROR_FLAG = 1
 
@@ -779,7 +781,7 @@ public:
 
 	RenderList render_list;
 
-	_FORCE_INLINE_ void _set_cull(bool p_front, bool p_reverse_cull);
+	_FORCE_INLINE_ void _set_cull(bool p_front, bool p_disabled, bool p_reverse_cull);
 
 	_FORCE_INLINE_ bool _setup_material(RasterizerStorageGLES3::Material *p_material, bool p_alpha_pass);
 	_FORCE_INLINE_ void _setup_geometry(RenderList::Element *e, const Transform &p_view_transform);

drivers/gles3/rasterizer_storage_gles3.cpp

@@ -4451,7 +4451,7 @@ RID RasterizerStorageGLES3::light_create(VS::LightType p_type) {
 	light->omni_shadow_mode = VS::LIGHT_OMNI_SHADOW_DUAL_PARABOLOID;
 	light->omni_shadow_detail = VS::LIGHT_OMNI_SHADOW_DETAIL_VERTICAL;
 	light->directional_blend_splits = false;
-
+	light->reverse_cull = false;
 	light->version = 0;
 
 	return light_owner.make_rid(light);
@@ -4530,6 +4530,14 @@ void RasterizerStorageGLES3::light_set_cull_mask(RID p_light, uint32_t p_mask) {
 	light->instance_change_notify();
 }
 
+void RasterizerStorageGLES3::light_set_reverse_cull_face_mode(RID p_light, bool p_enabled) {
+
+	Light *light = light_owner.getornull(p_light);
+	ERR_FAIL_COND(!light);
+
+	light->reverse_cull = p_enabled;
+}
+
 void RasterizerStorageGLES3::light_omni_set_shadow_mode(RID p_light, VS::LightOmniShadowMode p_mode) {
 
 	Light *light = light_owner.getornull(p_light);

drivers/gles3/rasterizer_storage_gles3.h

@@ -868,6 +868,7 @@ public:
 		RID projector;
 		bool shadow;
 		bool negative;
+		bool reverse_cull;
 		uint32_t cull_mask;
 		VS::LightOmniShadowMode omni_shadow_mode;
 		VS::LightOmniShadowDetail omni_shadow_detail;
@@ -887,6 +888,7 @@ public:
 	virtual void light_set_projector(RID p_light, RID p_texture);
 	virtual void light_set_negative(RID p_light, bool p_enable);
 	virtual void light_set_cull_mask(RID p_light, uint32_t p_mask);
+	virtual void light_set_reverse_cull_face_mode(RID p_light, bool p_enabled);
 
 	virtual void light_omni_set_shadow_mode(RID p_light, VS::LightOmniShadowMode p_mode);
 	virtual void light_omni_set_shadow_detail(RID p_light, VS::LightOmniShadowDetail p_detail);

scene/3d/light.cpp

@@ -106,6 +106,16 @@ Color Light::get_shadow_color() const {
 	return shadow_color;
 }
 
+void Light::set_shadow_reverse_cull_face(bool p_enable) {
+	reverse_cull = p_enable;
+	VS::get_singleton()->light_set_reverse_cull_face_mode(light, reverse_cull);
+}
+
+bool Light::get_shadow_reverse_cull_face() const {
+
+	return reverse_cull;
+}
+
 Rect3 Light::get_aabb() const {
 
 	if (type == VisualServer::LIGHT_DIRECTIONAL) {
@@ -202,6 +212,9 @@ void Light::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_color", "color"), &Light::set_color);
 	ClassDB::bind_method(D_METHOD("get_color"), &Light::get_color);
 
+	ClassDB::bind_method(D_METHOD("set_shadow_reverse_cull_face", "enable"), &Light::set_shadow_reverse_cull_face);
+	ClassDB::bind_method(D_METHOD("get_shadow_reverse_cull_face"), &Light::get_shadow_reverse_cull_face);
+
 	ClassDB::bind_method(D_METHOD("set_shadow_color", "shadow_color"), &Light::set_shadow_color);
 	ClassDB::bind_method(D_METHOD("get_shadow_color"), &Light::get_shadow_color);
 
@@ -217,6 +230,7 @@ void Light::_bind_methods() {
 	ADD_PROPERTYI(PropertyInfo(Variant::REAL, "shadow_bias", PROPERTY_HINT_RANGE, "-16,16,0.01"), "set_param", "get_param", PARAM_SHADOW_BIAS);
 	ADD_PROPERTYI(PropertyInfo(Variant::REAL, "shadow_contact", PROPERTY_HINT_RANGE, "0,16,0.01"), "set_param", "get_param", PARAM_CONTACT_SHADOW_SIZE);
 	ADD_PROPERTYI(PropertyInfo(Variant::REAL, "shadow_max_distance", PROPERTY_HINT_RANGE, "0,65536,0.1"), "set_param", "get_param", PARAM_SHADOW_MAX_DISTANCE);
+	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "shadow_reverse_cull_face"), "set_shadow_reverse_cull_face", "get_shadow_reverse_cull_face");
 	ADD_GROUP("Editor", "");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "editor_only"), "set_editor_only", "is_editor_only");
 	ADD_GROUP("", "");
@@ -244,6 +258,8 @@ Light::Light(VisualServer::LightType p_type) {
 	light = VisualServer::get_singleton()->light_create(p_type);
 	VS::get_singleton()->instance_set_base(get_instance(), light);
 
+	reverse_cull = false;
+
 	editor_only = false;
 	set_color(Color(1, 1, 1, 1));
 	set_shadow(false);

scene/3d/light.h

@@ -69,6 +69,7 @@ private:
 	Color shadow_color;
 	bool shadow;
 	bool negative;
+	bool reverse_cull;
 	uint32_t cull_mask;
 	VS::LightType type;
 	bool editor_only;
@@ -110,6 +111,9 @@ public:
 	void set_shadow_color(const Color &p_shadow_color);
 	Color get_shadow_color() const;
 
+	void set_shadow_reverse_cull_face(bool p_enable);
+	bool get_shadow_reverse_cull_face() const;
+
 	virtual Rect3 get_aabb() const;
 	virtual PoolVector<Face3> get_faces(uint32_t p_usage_flags) const;
 

scene/3d/visual_instance.cpp

@@ -231,14 +231,6 @@ void GeometryInstance::_notification(int p_what) {
 void GeometryInstance::set_flag(Flags p_flag, bool p_value) {
 
 	ERR_FAIL_INDEX(p_flag, FLAG_MAX);
-	if (p_flag == FLAG_CAST_SHADOW) {
-		if (p_value == true) {
-			set_cast_shadows_setting(SHADOW_CASTING_SETTING_ON);
-		} else {
-			set_cast_shadows_setting(SHADOW_CASTING_SETTING_OFF);
-		}
-	}
-
 	if (flags[p_flag] == p_value)
 		return;
 
@@ -252,14 +244,6 @@ bool GeometryInstance::get_flag(Flags p_flag) const {
 
 	ERR_FAIL_INDEX_V(p_flag, FLAG_MAX, false);
 
-	if (p_flag == FLAG_CAST_SHADOW) {
-		if (shadow_casting_setting == SHADOW_CASTING_SETTING_OFF) {
-			return false;
-		} else {
-			return true;
-		}
-	}
-
 	return flags[p_flag];
 }
 
@@ -330,7 +314,6 @@ void GeometryInstance::_bind_methods() {
 
 	//ADD_SIGNAL( MethodInfo("visibility_changed"));
 
-	BIND_CONSTANT(FLAG_CAST_SHADOW);
 	BIND_CONSTANT(FLAG_VISIBLE_IN_ALL_ROOMS);
 	BIND_CONSTANT(FLAG_MAX);
 
@@ -350,8 +333,6 @@ GeometryInstance::GeometryInstance() {
 		flags[i] = false;
 	}
 
-	flags[FLAG_CAST_SHADOW] = true;
-
 	shadow_casting_setting = SHADOW_CASTING_SETTING_ON;
 	extra_cull_margin = 0;
 	//VS::get_singleton()->instance_geometry_set_baked_light_texture_index(get_instance(),0);

scene/3d/visual_instance.h

@@ -84,7 +84,6 @@ class GeometryInstance : public VisualInstance {
 
 public:
 	enum Flags {
-		FLAG_CAST_SHADOW = VS::INSTANCE_FLAG_CAST_SHADOW,
 		FLAG_VISIBLE_IN_ALL_ROOMS = VS::INSTANCE_FLAG_VISIBLE_IN_ALL_ROOMS,
 		FLAG_USE_BAKED_LIGHT = VS::INSTANCE_FLAG_USE_BAKED_LIGHT,
 		FLAG_MAX = VS::INSTANCE_FLAG_MAX,

servers/visual/rasterizer.h

@@ -331,6 +331,7 @@ public:
 	virtual void light_set_projector(RID p_light, RID p_texture) = 0;
 	virtual void light_set_negative(RID p_light, bool p_enable) = 0;
 	virtual void light_set_cull_mask(RID p_light, uint32_t p_mask) = 0;
+	virtual void light_set_reverse_cull_face_mode(RID p_light, bool p_enabled) = 0;
 
 	virtual void light_omni_set_shadow_mode(RID p_light, VS::LightOmniShadowMode p_mode) = 0;
 	virtual void light_omni_set_shadow_detail(RID p_light, VS::LightOmniShadowDetail p_detail) = 0;

servers/visual/visual_server_raster.h

@@ -786,6 +786,7 @@ public:
 	BIND2(light_set_projector, RID, RID)
 	BIND2(light_set_negative, RID, bool)
 	BIND2(light_set_cull_mask, RID, uint32_t)
+	BIND2(light_set_reverse_cull_face_mode, RID, bool)
 
 	BIND2(light_omni_set_shadow_mode, RID, LightOmniShadowMode)
 	BIND2(light_omni_set_shadow_detail, RID, LightOmniShadowDetail)

servers/visual/visual_server_scene.cpp

@@ -978,16 +978,6 @@ void VisualServerScene::instance_geometry_set_flag(RID p_instance, VS::InstanceF
 
 	switch (p_flags) {
 
-		case VS::INSTANCE_FLAG_CAST_SHADOW: {
-			if (p_enabled == true) {
-				instance->cast_shadows = VS::SHADOW_CASTING_SETTING_ON;
-			} else {
-				instance->cast_shadows = VS::SHADOW_CASTING_SETTING_OFF;
-			}
-
-			instance->base_material_changed(); // to actually compute if shadows are visible or not
-
-		} break;
 		case VS::INSTANCE_FLAG_VISIBLE_IN_ALL_ROOMS: {
 
 			instance->visible_in_all_rooms = p_enabled;
@@ -1001,6 +991,12 @@ void VisualServerScene::instance_geometry_set_flag(RID p_instance, VS::InstanceF
 	}
 }
 void VisualServerScene::instance_geometry_set_cast_shadows_setting(RID p_instance, VS::ShadowCastingSetting p_shadow_casting_setting) {
+
+	Instance *instance = instance_owner.get(p_instance);
+	ERR_FAIL_COND(!instance);
+
+	instance->cast_shadows = p_shadow_casting_setting;
+	instance->base_material_changed(); // to actually compute if shadows are visible or not
 }
 void VisualServerScene::instance_geometry_set_material_override(RID p_instance, RID p_material) {
 

servers/visual/visual_server_wrap_mt.h

@@ -225,6 +225,7 @@ public:
 	FUNC2(light_set_projector, RID, RID)
 	FUNC2(light_set_negative, RID, bool)
 	FUNC2(light_set_cull_mask, RID, uint32_t)
+	FUNC2(light_set_reverse_cull_face_mode, RID, bool)
 
 	FUNC2(light_omni_set_shadow_mode, RID, LightOmniShadowMode)
 	FUNC2(light_omni_set_shadow_detail, RID, LightOmniShadowDetail)

servers/visual_server.h

@@ -371,6 +371,7 @@ public:
 	virtual void light_set_projector(RID p_light, RID p_texture) = 0;
 	virtual void light_set_negative(RID p_light, bool p_enable) = 0;
 	virtual void light_set_cull_mask(RID p_light, uint32_t p_mask) = 0;
+	virtual void light_set_reverse_cull_face_mode(RID p_light, bool p_enabled) = 0;
 
 	// omni light
 	enum LightOmniShadowMode {
@@ -743,7 +744,6 @@ public:
 	virtual Vector<ObjectID> instances_cull_convex(const Vector<Plane> &p_convex, RID p_scenario = RID()) const = 0;
 
 	enum InstanceFlags {
-		INSTANCE_FLAG_CAST_SHADOW,
 		INSTANCE_FLAG_VISIBLE_IN_ALL_ROOMS,
 		INSTANCE_FLAG_USE_BAKED_LIGHT,
 		INSTANCE_FLAG_MAX