New boolean: Fix second example in T79404.

The code to detect non-trivial coplanar intersection sometimes
falsely said there wasn't one. This caused some coplanar intersections
to be missed. Also added a test for this case.

2 files changed, 29 insertions, 2 deletions

diff --git a/source/blender/blenlib/intern/mesh_intersect.cc b/source/blender/blenlib/intern/mesh_intersect.cc
index 5bd25404674..c36dfa80be7 100644
--- a/source/blender/blenlib/intern/mesh_intersect.cc
+++ b/source/blender/blenlib/intern/mesh_intersect.cc
@@ -1130,7 +1130,7 @@ static bool non_trivially_2d_point_in_tri(const int orients[3][3], int pi)
  * one gets by having, each point of one triangle being strictly right-of one
  * edge of the other and strictly left of the other two edges; and vice versa.
  * In addition, it must not be the case that all of the points of one triangle
- * are totally to one side of one edge of the other triangle, and vice versa.
+ * are totally on the outside of one edge of the other triangle, and vice versa.
  */
 static bool non_trivially_2d_hex_overlap(int orients[2][3][3])
 {
@@ -1142,7 +1142,7 @@ static bool non_trivially_2d_hex_overlap(int orients[2][3][3])
         return false;
       }
       int s = orients[ab][0][i] + orients[ab][1][i] + orients[ab][2][i];
-      if (s == 3 || s == -3) {
+      if (s == -3) {
         return false;
       }
     }
diff --git a/source/blender/blenlib/tests/BLI_mesh_intersect_test.cc b/source/blender/blenlib/tests/BLI_mesh_intersect_test.cc
index a8f05eedaaa..7b0f05444e2 100644
--- a/source/blender/blenlib/tests/BLI_mesh_intersect_test.cc
+++ b/source/blender/blenlib/tests/BLI_mesh_intersect_test.cc
@@ -703,6 +703,33 @@ TEST(mesh_intersect, CubeCubeStep)
     write_obj_mesh(out2, "test_cubecubestep_nary");
   }
 }
+
+TEST(mesh_intersect, RectCross)
+{
+  const char *spec = R"(8 4
+  3/2 0 1
+  -3/2 0 1
+  -3/2 0 -1
+  3/2 0 -1
+  1 0 -5
+  -1 0 -5
+  1 0 5
+  -1 0 5
+  1 0 3
+  1 3 2
+  5 4 6
+  5 6 7
+  )";
+
+  IMeshBuilder mb(spec);
+  IMesh out = trimesh_self_intersect(mb.imesh, &mb.arena);
+  out.populate_vert();
+  EXPECT_EQ(out.vert_size(), 17);
+  EXPECT_EQ(out.face_size(), 28);
+  if (DO_OBJ) {
+    write_obj_mesh(out, "test_rectcross");
+  }
+}
 #  endif
 
 #  if DO_PERF_TESTS