From 70cd25f3eba286efa6149d0ccc267b3fb2cb91d0 Mon Sep 17 00:00:00 2001
From: lawnjelly <lawnjelly@gmail.com>
Date: Wed, 20 Mar 2024 14:11:36 +0000
Subject: [PATCH] Tighter shadow culling - fix light colinear to frustum edge

In rare situations if a light is placed near colinear to a frustum edge, the extra culling plane derived can have an inaccurate normal due to floating point error.
This PR detects colinear triangles, and prevents adding a culling plane in this situation.
---
 servers/rendering/rendering_light_culler.cpp | 18 +++++++----
 servers/rendering/rendering_light_culler.h   | 33 ++++++++++++++++++++
 2 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/servers/rendering/rendering_light_culler.cpp b/servers/rendering/rendering_light_culler.cpp
index fdf0d73654c..0889898f0b6 100644
--- a/servers/rendering/rendering_light_culler.cpp
+++ b/servers/rendering/rendering_light_culler.cpp
@@ -427,15 +427,19 @@ bool RenderingLightCuller::_add_light_camera_planes(LightCullPlanes &r_cull_plan
 	uint8_t *entry = &data.LUT_entries[lookup][0];
 	int n_edges = data.LUT_entry_sizes[lookup] - 1;
 
+	const Vector3 &pt2 = p_light_source.pos;
+
 	for (int e = 0; e < n_edges; e++) {
 		int i0 = entry[e];
 		int i1 = entry[e + 1];
 		const Vector3 &pt0 = data.frustum_points[i0];
 		const Vector3 &pt1 = data.frustum_points[i1];
 
-		// Create plane from 3 points.
-		Plane p(pt0, pt1, p_light_source.pos);
-		r_cull_planes.add_cull_plane(p);
+		if (!_is_colinear_tri(pt0, pt1, pt2)) {
+			// Create plane from 3 points.
+			Plane p(pt0, pt1, pt2);
+			r_cull_planes.add_cull_plane(p);
+		}
 	}
 
 	// Last to 0 edge.
@@ -446,9 +450,11 @@ bool RenderingLightCuller::_add_light_camera_planes(LightCullPlanes &r_cull_plan
 		const Vector3 &pt0 = data.frustum_points[i0];
 		const Vector3 &pt1 = data.frustum_points[i1];
 
-		// Create plane from 3 points.
-		Plane p(pt0, pt1, p_light_source.pos);
-		r_cull_planes.add_cull_plane(p);
+		if (!_is_colinear_tri(pt0, pt1, pt2)) {
+			// Create plane from 3 points.
+			Plane p(pt0, pt1, pt2);
+			r_cull_planes.add_cull_plane(p);
+		}
 	}
 
 #ifdef LIGHT_CULLER_DEBUG_LOGGING
diff --git a/servers/rendering/rendering_light_culler.h b/servers/rendering/rendering_light_culler.h
index 602543850aa..0bf975430b2 100644
--- a/servers/rendering/rendering_light_culler.h
+++ b/servers/rendering/rendering_light_culler.h
@@ -163,6 +163,39 @@ private:
 
 	bool _prepare_light(const RendererSceneCull::Instance &p_instance, int32_t p_directional_light_id = -1);
 
+	// Avoid adding extra culling planes derived from near colinear triangles.
+	// The normals derived from these will be inaccurate, and can lead to false
+	// culling of objects that should be within the light volume.
+	bool _is_colinear_tri(const Vector3 &p_a, const Vector3 &p_b, const Vector3 &p_c) const {
+		// Lengths of sides a, b and c.
+		float la = (p_b - p_a).length();
+		float lb = (p_c - p_b).length();
+		float lc = (p_c - p_a).length();
+
+		// Get longest side into lc.
+		if (lb < la) {
+			SWAP(la, lb);
+		}
+		if (lc < lb) {
+			SWAP(lb, lc);
+		}
+
+		// Prevent divide by zero.
+		if (lc > 0.00001f) {
+			// If the summed length of the smaller two
+			// sides is close to the length of the longest side,
+			// the points are colinear, and the triangle is near degenerate.
+			float ld = ((la + lb) - lc) / lc;
+
+			// ld will be close to zero for colinear tris.
+			return ld < 0.00001f;
+		}
+
+		// Don't create planes from tiny triangles,
+		// they won't be accurate.
+		return true;
+	}
+
 	// Internal version uses LightSource.
 	bool _add_light_camera_planes(LightCullPlanes &r_cull_planes, const LightSource &p_light_source);