Fix CSG edge case causing intersection line to hit on common edge of 2 triangles.

The previous implementation assumed that the intersection entered or exited a
shape when it hit right on the common edge of 2 triangles. However, there is
also a case where it just "skirts" the other shape on the outside.

To fix this, we added code to check the intersection distance and if the
normals of the faces are pointed in the same direction as the intersection or
not (e.g. inner product > 0). This handles the case where the intersection
line hits the common edge of 2 triangles and skirts the other shape on the
outside.

Extended code to cover a third case.

Fixes #58637.

Co-authored-by: OldBelge <StevenGeens@users.noreply.github.com>

modules/csg/csg.cpp

@@ -526,17 +526,19 @@ int CSGBrushOperation::MeshMerge::_create_bvh(FaceBVH *facebvhptr, FaceBVH **fac
 	return index;
 }
 
-void CSGBrushOperation::MeshMerge::_add_distance(List<real_t> &r_intersectionsA, List<real_t> &r_intersectionsB, bool p_from_B, real_t p_distance) const {
+void CSGBrushOperation::MeshMerge::_add_distance(List<IntersectionDistance> &r_intersectionsA, List<IntersectionDistance> &r_intersectionsB, bool p_from_B, real_t p_distance_squared, bool p_is_conormal) const {
-	List<real_t> &intersections = p_from_B ? r_intersectionsB : r_intersectionsA;
+	List<IntersectionDistance> &intersections = p_from_B ? r_intersectionsB : r_intersectionsA;
 
 	// Check if distance exists.
-	for (const real_t E : intersections) {
+	for (const IntersectionDistance E : intersections) {
-		if (Math::is_equal_approx(E, p_distance)) {
+		if (E.is_conormal == p_is_conormal && Math::is_equal_approx(E.distance_squared, p_distance_squared)) {
 			return;
 		}
 	}
-
+	IntersectionDistance IntersectionDistance;
-	intersections.push_back(p_distance);
+	IntersectionDistance.is_conormal = p_is_conormal;
+	IntersectionDistance.distance_squared = p_distance_squared;
+	intersections.push_back(IntersectionDistance);
 }
 
 bool CSGBrushOperation::MeshMerge::_bvh_inside(FaceBVH *facebvhptr, int p_max_depth, int p_bvh_first, int p_face_idx) const {
@@ -561,8 +563,11 @@ bool CSGBrushOperation::MeshMerge::_bvh_inside(FaceBVH *facebvhptr, int p_max_de
 		VISITED_BIT_MASK = ~NODE_IDX_MASK
 	};
 
-	List<real_t> intersectionsA;
+	List<IntersectionDistance> intersectionsA;
-	List<real_t> intersectionsB;
+	List<IntersectionDistance> intersectionsB;
+
+	Intersection closest_intersection;
+	closest_intersection.found = false;
 
 	int level = 0;
 	int pos = p_bvh_first;
@@ -587,17 +592,61 @@ bool CSGBrushOperation::MeshMerge::_bvh_inside(FaceBVH *facebvhptr, int p_max_de
 							};
 							Vector3 current_normal = Plane(current_points[0], current_points[1], current_points[2]).normal;
 							Vector3 intersection_point;
-
 							// Check if faces are co-planar.
 							if (current_normal.is_equal_approx(face_normal) &&
 									is_point_in_triangle(face_center, current_points)) {
 								// Only add an intersection if not a B face.
 								if (!face.from_b) {
-									_add_distance(intersectionsA, intersectionsB, current_face.from_b, 0);
+									_add_distance(intersectionsA, intersectionsB, current_face.from_b, 0, true);
 								}
 							} else if (ray_intersects_triangle(face_center, face_normal, current_points, CMP_EPSILON, intersection_point)) {
-								real_t distance = face_center.distance_to(intersection_point);
+								real_t distance_squared = face_center.distance_squared_to(intersection_point);
-								_add_distance(intersectionsA, intersectionsB, current_face.from_b, distance);
+								real_t inner = current_normal.dot(face_normal);
+								// If the faces are perpendicular, ignore this face.
+								// The triangles on the side should be intersected and result in the correct behavior.
+								if (!Math::is_zero_approx(inner)) {
+									_add_distance(intersectionsA, intersectionsB, current_face.from_b, distance_squared, inner > 0.0f);
+								}
+							}
+
+							if (face.from_b != current_face.from_b) {
+								if (current_normal.is_equal_approx(face_normal) &&
+										is_point_in_triangle(face_center, current_points)) {
+									// Only add an intersection if not a B face.
+									if (!face.from_b) {
+										closest_intersection.found = true;
+										closest_intersection.conormal = 1.0f;
+										closest_intersection.distance_squared = 0.0f;
+										closest_intersection.origin_angle = -FLT_MAX;
+									}
+								} else if (ray_intersects_triangle(face_center, face_normal, current_points, CMP_EPSILON, intersection_point)) {
+									Intersection potential_intersection;
+									potential_intersection.found = true;
+									potential_intersection.conormal = face_normal.dot(current_normal);
+									potential_intersection.distance_squared = face_center.distance_squared_to(intersection_point);
+									potential_intersection.origin_angle = Math::abs(potential_intersection.conormal);
+									real_t intersection_dist_from_face = face_normal.dot(intersection_point - face_center);
+									for (int i = 0; i < 3; i++) {
+										real_t point_dist_from_face = face_normal.dot(current_points[i] - face_center);
+										if (!Math::is_equal_approx(point_dist_from_face, intersection_dist_from_face) &&
+												point_dist_from_face < intersection_dist_from_face) {
+											potential_intersection.origin_angle = -potential_intersection.origin_angle;
+											break;
+										}
+									}
+									if (potential_intersection.conormal != 0.0f) {
+										if (!closest_intersection.found) {
+											closest_intersection = potential_intersection;
+										} else if (!Math::is_equal_approx(potential_intersection.distance_squared, closest_intersection.distance_squared) &&
+												potential_intersection.distance_squared < closest_intersection.distance_squared) {
+											closest_intersection = potential_intersection;
+										} else if (Math::is_equal_approx(potential_intersection.distance_squared, closest_intersection.distance_squared)) {
+											if (potential_intersection.origin_angle < closest_intersection.origin_angle) {
+												closest_intersection = potential_intersection;
+											}
+										}
+									}
+								}
 							}
 						}
 
@@ -652,8 +701,11 @@ bool CSGBrushOperation::MeshMerge::_bvh_inside(FaceBVH *facebvhptr, int p_max_de
 		}
 	}
 
-	// Inside if face normal intersects other faces an odd number of times.
+	if (!closest_intersection.found) {
-	return (intersectionsA.size() + intersectionsB.size()) & 1;
+		return false;
+	} else {
+		return closest_intersection.conormal > 0.0f;
+	}
 }
 
 void CSGBrushOperation::MeshMerge::mark_inside_faces() {

modules/csg/csg.h

@@ -135,6 +135,17 @@ struct CSGBrushOperation {
 				return h;
 			}
 		};
+		struct Intersection {
+			bool found = false;
+			real_t conormal = FLT_MAX;
+			real_t distance_squared = FLT_MAX;
+			real_t origin_angle = FLT_MAX;
+		};
+
+		struct IntersectionDistance {
+			bool is_conormal;
+			real_t distance_squared;
+		};
 
 		Vector<Vector3> points;
 		Vector<Face> faces;
@@ -143,7 +154,7 @@ struct CSGBrushOperation {
 		OAHashMap<VertexKey, int, VertexKeyHash> snap_cache;
 		float vertex_snap = 0.0;
 
-		inline void _add_distance(List<real_t> &r_intersectionsA, List<real_t> &r_intersectionsB, bool p_from_B, real_t p_distance) const;
+		inline void _add_distance(List<IntersectionDistance> &r_intersectionsA, List<IntersectionDistance> &r_intersectionsB, bool p_from_B, real_t p_distance, bool p_is_conormal) const;
 		inline bool _bvh_inside(FaceBVH *facebvhptr, int p_max_depth, int p_bvh_first, int p_face_idx) const;
 		inline int _create_bvh(FaceBVH *facebvhptr, FaceBVH **facebvhptrptr, int p_from, int p_size, int p_depth, int &r_max_depth, int &r_max_alloc);
 

modules/csg/csg_shape.cpp

@@ -653,7 +653,7 @@ void CSGShape3D::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("get_meshes"), &CSGShape3D::get_meshes);
 
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "operation", PROPERTY_HINT_ENUM, "Union,Intersection,Subtraction"), "set_operation", "get_operation");
-	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "snap", PROPERTY_HINT_RANGE, "0.0001,1,0.001,suffix:m"), "set_snap", "get_snap");
+	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "snap", PROPERTY_HINT_RANGE, "0.000001,1,0.000001,suffix:m"), "set_snap", "get_snap");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "calculate_tangents"), "set_calculate_tangents", "is_calculating_tangents");
 
 	ADD_GROUP("Collision", "collision_");

modules/csg/doc_classes/CSGShape3D.xml

@@ -73,7 +73,7 @@
 			The operation that is performed on this shape. This is ignored for the first CSG child node as the operation is between this node and the previous child of this nodes parent.
 		</member>
 		<member name="snap" type="float" setter="set_snap" getter="get_snap" default="0.001">
-			Snap makes the mesh snap to a given distance so that the faces of two meshes can be perfectly aligned. A lower value results in greater precision but may be harder to adjust.
+			Snap makes the mesh vertices snap to a given distance so that the faces of two meshes can be perfectly aligned. A lower value results in greater precision but may be harder to adjust.
 		</member>
 		<member name="use_collision" type="bool" setter="set_use_collision" getter="is_using_collision" default="false">
 			Adds a collision shape to the physics engine for our CSG shape. This will always act like a static body. Note that the collision shape is still active even if the CSG shape itself is hidden. See also [member collision_mask] and [member collision_priority].