Fluid: Optimization for mesh file loading

Improved loading times for mesh files by reading bigger chunks of data from the disk at once.

1 files changed, 125 insertions, 33 deletions

diff --git a/intern/mantaflow/intern/MANTA_main.cpp b/intern/mantaflow/intern/MANTA_main.cpp
index 2b3a282d815..bf58e22938a 100644
--- a/intern/mantaflow/intern/MANTA_main.cpp
+++ b/intern/mantaflow/intern/MANTA_main.cpp
@@ -54,6 +54,10 @@ int MANTA::with_debug(0);
 
 /* Number of particles that the cache reads at once (with zlib). */
 #define PARTICLE_CHUNK 20000
+/* Number of mesh nodes that the cache reads at once (with zlib). */
+#define NODE_CHUNK 20000
+/* Number of mesh triangles that the cache reads at once (with zlib). */
+#define TRIANGLE_CHUNK 20000
 
 MANTA::MANTA(int *res, FluidModifierData *mmd) : mCurrentID(++solverID)
 {
@@ -2461,29 +2465,59 @@ void MANTA::updateMeshFromBobj(const char *filename)
     std::cout << "MANTA::updateMeshFromBobj()" << std::endl;
 
   gzFile gzf;
-  float fbuffer[3];
-  int ibuffer[3];
-  int numBuffer = 0;
 
   gzf = (gzFile)BLI_gzopen(filename, "rb1");  // do some compression
   if (!gzf)
     std::cerr << "updateMeshData: unable to open file: " << filename << std::endl;
 
+  int numBuffer = 0;
+
   // Num vertices
   gzread(gzf, &numBuffer, sizeof(int));
 
   if (with_debug)
     std::cout << "read mesh , num verts: " << numBuffer << " , in file: " << filename << std::endl;
 
+  int numChunks = (int)(ceil((float)numBuffer / NODE_CHUNK));
+  int readLen, readStart, readEnd, readBytes, k;
+
   if (numBuffer) {
     // Vertices
+    int todoVertices = numBuffer;
+    float *bufferVerts = (float *)MEM_malloc_arrayN(
+        NODE_CHUNK, sizeof(float) * 3, "fluid_mesh_vertices");
+
     mMeshNodes->resize(numBuffer);
-    for (std::vector<Node>::iterator it = mMeshNodes->begin(); it != mMeshNodes->end(); ++it) {
-      gzread(gzf, fbuffer, sizeof(float) * 3);
-      it->pos[0] = fbuffer[0];
-      it->pos[1] = fbuffer[1];
-      it->pos[2] = fbuffer[2];
+
+    for (int i = 0; i < numChunks && todoVertices > 0; ++i) {
+      readLen = NODE_CHUNK;
+      if (todoVertices < NODE_CHUNK) {
+        readLen = todoVertices;
+      }
+
+      readBytes = gzread(gzf, bufferVerts, readLen * sizeof(float) * 3);
+      if (!readBytes) {
+        if (with_debug)
+          std::cerr << "error while reading vertices" << std::endl;
+        MEM_freeN(bufferVerts);
+        gzclose(gzf);
+        return;
+      }
+
+      readStart = (numBuffer - todoVertices);
+      CLAMP(readStart, 0, numBuffer);
+      readEnd = readStart + readLen;
+      CLAMP(readEnd, 0, numBuffer);
+
+      k = 0;
+      for (std::vector<MANTA::Node>::size_type j = readStart; j < readEnd; j++, k += 3) {
+        mMeshNodes->at(j).pos[0] = bufferVerts[k];
+        mMeshNodes->at(j).pos[1] = bufferVerts[k + 1];
+        mMeshNodes->at(j).pos[2] = bufferVerts[k + 2];
+      }
+      todoVertices -= readLen;
     }
+    MEM_freeN(bufferVerts);
   }
 
   // Num normals
@@ -2495,14 +2529,42 @@ void MANTA::updateMeshFromBobj(const char *filename)
 
   if (numBuffer) {
     // Normals
+    int todoNormals = numBuffer;
+    float *bufferNormals = (float *)MEM_malloc_arrayN(
+        NODE_CHUNK, sizeof(float) * 3, "fluid_mesh_normals");
+
     if (!getNumVertices())
       mMeshNodes->resize(numBuffer);
-    for (std::vector<Node>::iterator it = mMeshNodes->begin(); it != mMeshNodes->end(); ++it) {
-      gzread(gzf, fbuffer, sizeof(float) * 3);
-      it->normal[0] = fbuffer[0];
-      it->normal[1] = fbuffer[1];
-      it->normal[2] = fbuffer[2];
+
+    for (int i = 0; i < numChunks && todoNormals > 0; ++i) {
+      readLen = NODE_CHUNK;
+      if (todoNormals < NODE_CHUNK) {
+        readLen = todoNormals;
+      }
+
+      readBytes = gzread(gzf, bufferNormals, readLen * sizeof(float) * 3);
+      if (!readBytes) {
+        if (with_debug)
+          std::cerr << "error while reading normals" << std::endl;
+        MEM_freeN(bufferNormals);
+        gzclose(gzf);
+        return;
+      }
+
+      readStart = (numBuffer - todoNormals);
+      CLAMP(readStart, 0, numBuffer);
+      readEnd = readStart + readLen;
+      CLAMP(readEnd, 0, numBuffer);
+
+      k = 0;
+      for (std::vector<MANTA::Node>::size_type j = readStart; j < readEnd; j++, k += 3) {
+        mMeshNodes->at(j).normal[0] = bufferNormals[k];
+        mMeshNodes->at(j).normal[1] = bufferNormals[k + 1];
+        mMeshNodes->at(j).normal[2] = bufferNormals[k + 2];
+      }
+      todoNormals -= readLen;
     }
+    MEM_freeN(bufferNormals);
   }
 
   // Num triangles
@@ -2512,18 +2574,45 @@ void MANTA::updateMeshFromBobj(const char *filename)
     std::cout << "read mesh , num triangles : " << numBuffer << " , in file: " << filename
               << std::endl;
 
+  numChunks = (int)(ceil((float)numBuffer / TRIANGLE_CHUNK));
+
   if (numBuffer) {
     // Triangles
+    int todoTriangles = numBuffer;
+    int *bufferTriangles = (int *)MEM_malloc_arrayN(
+        TRIANGLE_CHUNK, sizeof(int) * 3, "fluid_mesh_triangles");
+
     mMeshTriangles->resize(numBuffer);
-    MANTA::Triangle *bufferTriangle;
-    for (std::vector<Triangle>::iterator it = mMeshTriangles->begin(); it != mMeshTriangles->end();
-         ++it) {
-      gzread(gzf, ibuffer, sizeof(int) * 3);
-      bufferTriangle = (MANTA::Triangle *)ibuffer;
-      it->c[0] = bufferTriangle->c[0];
-      it->c[1] = bufferTriangle->c[1];
-      it->c[2] = bufferTriangle->c[2];
+
+    for (int i = 0; i < numChunks && todoTriangles > 0; ++i) {
+      readLen = TRIANGLE_CHUNK;
+      if (todoTriangles < TRIANGLE_CHUNK) {
+        readLen = todoTriangles;
+      }
+
+      readBytes = gzread(gzf, bufferTriangles, readLen * sizeof(int) * 3);
+      if (!readBytes) {
+        if (with_debug)
+          std::cerr << "error while reading triangles" << std::endl;
+        MEM_freeN(bufferTriangles);
+        gzclose(gzf);
+        return;
+      }
+
+      readStart = (numBuffer - todoTriangles);
+      CLAMP(readStart, 0, numBuffer);
+      readEnd = readStart + readLen;
+      CLAMP(readEnd, 0, numBuffer);
+
+      k = 0;
+      for (std::vector<MANTA::Triangle>::size_type j = readStart; j < readEnd; j++, k += 3) {
+        mMeshTriangles->at(j).c[0] = bufferTriangles[k];
+        mMeshTriangles->at(j).c[1] = bufferTriangles[k + 1];
+        mMeshTriangles->at(j).c[2] = bufferTriangles[k + 2];
+      }
+      todoTriangles -= readLen;
     }
+    MEM_freeN(bufferTriangles);
   }
   gzclose(gzf);
 }
@@ -2767,7 +2856,7 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
   numParticles = ibuffer[0];
 
   const int numChunks = (int)(ceil((float)numParticles / PARTICLE_CHUNK));
-  int todoParticles, readLen, k;
+  int todoParticles, readLen;
   int readStart, readEnd, readBytes;
 
   // Reading base particle system file v2
@@ -2775,7 +2864,7 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
     MANTA::pData *bufferPData;
     todoParticles = numParticles;
     bufferPData = (MANTA::pData *)MEM_malloc_arrayN(
-        PARTICLE_CHUNK, sizeof(pData), "fluid_particle_data");
+        PARTICLE_CHUNK, sizeof(MANTA::pData), "fluid_particle_data");
 
     dataPointer->resize(numParticles);
 
@@ -2796,16 +2885,17 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
 
       readStart = (numParticles - todoParticles);
       CLAMP(readStart, 0, numParticles);
-      readEnd = readStart + PARTICLE_CHUNK;
+      readEnd = readStart + readLen;
       CLAMP(readEnd, 0, numParticles);
 
-      for (std::vector<pData>::size_type j = readStart, k = 0; j < readEnd; j++, k++) {
+      int k = 0;
+      for (std::vector<MANTA::pData>::size_type j = readStart; j < readEnd; j++, k++) {
         dataPointer->at(j).pos[0] = bufferPData[k].pos[0];
         dataPointer->at(j).pos[1] = bufferPData[k].pos[1];
         dataPointer->at(j).pos[2] = bufferPData[k].pos[2];
         dataPointer->at(j).flag = bufferPData[k].flag;
       }
-      todoParticles -= PARTICLE_CHUNK;
+      todoParticles -= readLen;
     }
     MEM_freeN(bufferPData);
   }
@@ -2814,7 +2904,7 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
     MANTA::pVel *bufferPVel;
     todoParticles = numParticles;
     bufferPVel = (MANTA::pVel *)MEM_malloc_arrayN(
-        PARTICLE_CHUNK, sizeof(pVel), "fluid_particle_velocity");
+        PARTICLE_CHUNK, sizeof(MANTA::pVel), "fluid_particle_velocity");
 
     velocityPointer->resize(numParticles);
 
@@ -2835,15 +2925,16 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
 
       readStart = (numParticles - todoParticles);
       CLAMP(readStart, 0, numParticles);
-      readEnd = readStart + PARTICLE_CHUNK;
+      readEnd = readStart + readLen;
       CLAMP(readEnd, 0, numParticles);
 
-      for (std::vector<pVel>::size_type j = readStart, k = 0; j < readEnd; j++, k++) {
+      int k = 0;
+      for (std::vector<MANTA::pVel>::size_type j = readStart; j < readEnd; j++, k++) {
         velocityPointer->at(j).pos[0] = bufferPVel[k].pos[0];
         velocityPointer->at(j).pos[1] = bufferPVel[k].pos[1];
         velocityPointer->at(j).pos[2] = bufferPVel[k].pos[2];
       }
-      todoParticles -= PARTICLE_CHUNK;
+      todoParticles -= readLen;
     }
     MEM_freeN(bufferPVel);
   }
@@ -2872,13 +2963,14 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
 
       readStart = (numParticles - todoParticles);
       CLAMP(readStart, 0, numParticles);
-      readEnd = readStart + PARTICLE_CHUNK;
+      readEnd = readStart + readLen;
       CLAMP(readEnd, 0, numParticles);
 
-      for (std::vector<pVel>::size_type j = readStart, k = 0; j < readEnd; j++, k++) {
+      int k = 0;
+      for (std::vector<float>::size_type j = readStart; j < readEnd; j++, k++) {
         lifePointer->at(j) = bufferPLife[k];
       }
-      todoParticles -= PARTICLE_CHUNK;
+      todoParticles -= readLen;
     }
     MEM_freeN(bufferPLife);
   }