Fix T76535: Eevee + Freestyle render crash with many strokes

Freestyle would create a huge amount of material slots with the same material,
causing issues in Eevee use of alloca().

2 files changed, 24 insertions, 12 deletions

diff --git a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
index b8f3a27be9c..ca956dfbbbf 100644
--- a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
+++ b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
@@ -25,6 +25,8 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "BLI_map.h"
+
 extern "C" {
 #include "RNA_access.h"
 #include "RNA_types.h"
@@ -463,7 +465,7 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const
   vector<StrokeGroup *> *groups = hasTex ? &self->texturedStrokeGroups : &self->strokeGroups;
   StrokeGroup *group;
   if (groups->empty() || !(groups->back()->totvert + totvert < MESH_MAX_VERTS &&
-                           groups->back()->totcol + 1 < MAXMAT)) {
+                           groups->back()->materials.size() + 1 < MAXMAT)) {
     group = new StrokeGroup;
     groups->push_back(group);
   }
@@ -475,7 +477,10 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const
   group->totedge += totedge;
   group->totpoly += totpoly;
   group->totloop += totloop;
-  group->totcol++;
+
+  if (!group->materials.contains(ma)) {
+    group->materials.add_new(ma, group->materials.size());
+  }
 }
 
 // Check if the triangle is visible (i.e., within the render image boundary)
@@ -587,7 +592,7 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
   mesh->totedge = group->totedge;
   mesh->totpoly = group->totpoly;
   mesh->totloop = group->totloop;
-  mesh->totcol = group->totcol;
+  mesh->totcol = group->materials.size();
 
   mesh->mvert = (MVert *)CustomData_add_layer(
       &mesh->vdata, CD_MVERT, CD_CALLOC, NULL, mesh->totvert);
@@ -628,12 +633,20 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
   mesh->mloopcol = colors;
 
   mesh->mat = (Material **)MEM_mallocN(sizeof(Material *) * mesh->totcol, "MaterialList");
+  for (const auto &item : group->materials.items()) {
+    Material *material = item.key;
+    const int matnr = item.value;
+    mesh->mat[matnr] = material;
+    if (material) {
+      id_us_plus(&material->id);
+    }
+  }
 
   ////////////////////
   //  Data copy
   ////////////////////
 
-  int vertex_index = 0, edge_index = 0, loop_index = 0, material_index = 0;
+  int vertex_index = 0, edge_index = 0, loop_index = 0;
   int visible_faces, visible_segments;
   bool visible;
   Strip::vertex_container::iterator v[3];
@@ -644,8 +657,7 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
                                            itend = group->strokes.end();
        it != itend;
        ++it) {
-    mesh->mat[material_index] = (*it)->getMaterial();
-    id_us_plus(&mesh->mat[material_index]->id);
+    const int matnr = group->materials.lookup_default((*it)->getMaterial(), 0);
 
     vector<Strip *> &strips = (*it)->getStrips();
     for (vector<Strip *>::const_iterator s = strips.begin(), send = strips.end(); s != send; ++s) {
@@ -727,7 +739,7 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
           // poly
           polys->loopstart = loop_index;
           polys->totloop = 3;
-          polys->mat_nr = material_index;
+          polys->mat_nr = matnr;
           ++polys;
 
           // Even and odd loops connect triangles vertices differently
@@ -812,8 +824,7 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
         }
       }  // loop over strip vertices
     }    // loop over strips
-    material_index++;
-  }  // loop over strokes
+  }      // loop over strokes
 
   BKE_object_materials_test(freestyle_bmain, object_mesh, (ID *)mesh);
 
@@ -821,7 +832,6 @@ void BlenderStrokeRenderer::GenerateStrokeMesh(StrokeGroup *group, bool hasTex)
   BLI_assert(mesh->totvert == vertex_index);
   BLI_assert(mesh->totedge == edge_index);
   BLI_assert(mesh->totloop == loop_index);
-  BLI_assert(mesh->totcol == material_index);
   BKE_mesh_validate(mesh, true, true);
 #endif
 }
diff --git a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
index c333319ada1..9c94228414d 100644
--- a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
+++ b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
@@ -21,6 +21,8 @@
  * \ingroup freestyle
  */
 
+#include "BLI_map.h"
+
 #include "../stroke/StrokeRenderer.h"
 #include "../system/FreestyleConfig.h"
 
@@ -50,15 +52,15 @@ class BlenderStrokeRenderer : public StrokeRenderer {
   Object *NewMesh() const;
 
   struct StrokeGroup {
-    explicit StrokeGroup() : totvert(0), totedge(0), totpoly(0), totloop(0), totcol(0)
+    explicit StrokeGroup() : totvert(0), totedge(0), totpoly(0), totloop(0)
     {
     }
     vector<StrokeRep *> strokes;
+    BLI::Map<Material *, int> materials;
     int totvert;
     int totedge;
     int totpoly;
     int totloop;
-    int totcol;
   };
   vector<StrokeGroup *> strokeGroups, texturedStrokeGroups;