Fluid: Updated Mantaflow source files

Includes improvements for the file IO. Namely, more meta data will be written from now on.

This change is required to prevent IO issues (e.g. T84649) that arised through the use of sparse grids caching (introduced in 2.92).

10 files changed, 212 insertions, 22 deletions

diff --git a/extern/mantaflow/preprocessed/fileio/iogrids.cpp b/extern/mantaflow/preprocessed/fileio/iogrids.cpp
index 7cedbb484ba..d138cd2925f 100644
--- a/extern/mantaflow/preprocessed/fileio/iogrids.cpp
+++ b/extern/mantaflow/preprocessed/fileio/iogrids.cpp
@@ -628,13 +628,24 @@ template<class T> int readGridUni(const string &name, Grid<T> *grid)
     // current file format
     UniHeader head;
     assertMsg(gzread(gzf, &head, sizeof(UniHeader)) == sizeof(UniHeader),
-              "can't read file, no header present");
-    assertMsg(head.dimX == grid->getSizeX() && head.dimY == grid->getSizeY() &&
-                  head.dimZ == grid->getSizeZ(),
-              "grid dim doesn't match, " << Vec3(head.dimX, head.dimY, head.dimZ) << " vs "
-                                         << grid->getSize());
+              "readGridUni: Can't read file, no header present");
     assertMsg(unifyGridType(head.gridType) == unifyGridType(grid->getType()),
-              "grid type doesn't match " << head.gridType << " vs " << grid->getType());
+              "readGridUni: Grid type doesn't match " << head.gridType << " vs "
+                                                      << grid->getType());
+
+    const Vec3i curGridSize = grid->getParent()->getGridSize();
+    const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
+#  if BLENDER
+    // Correct grid size is only a soft requirement in Blender
+    if (headGridSize != curGridSize) {
+      debMsg("readGridUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
+      return 0;
+    }
+#  else
+    assertMsg(headGridSize == curGridSize,
+              "readGridUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
+#  endif
+
 #  if FLOATINGPOINT_PRECISION != 1
     // convert float to double
     Grid<T> temp(grid->getParent());
diff --git a/extern/mantaflow/preprocessed/fileio/ioparticles.cpp b/extern/mantaflow/preprocessed/fileio/ioparticles.cpp
index 36e10aa1644..538d42f3d87 100644
--- a/extern/mantaflow/preprocessed/fileio/ioparticles.cpp
+++ b/extern/mantaflow/preprocessed/fileio/ioparticles.cpp
@@ -230,6 +230,19 @@ int readParticlesUni(const std::string &name, BasicParticleSystem *parts)
     assertMsg(((head.bytesPerElement == PartSysSize) && (head.elementType == 0)),
               "particle type doesn't match");
 
+    const Vec3i curGridSize = parts->getParent()->getGridSize();
+    const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
+#  if BLENDER
+    // Correct grid size is only a soft requirement in Blender
+    if (headGridSize != curGridSize) {
+      debMsg("readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
+      return 0;
+    }
+#  else
+    assertMsg(headGridSize == curGridSize,
+              "readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
+#  endif
+
     // re-allocate all data
     parts->resizeAll(head.dim);
 
@@ -325,6 +338,19 @@ template<class T> int readPdataUni(const std::string &name, ParticleDataImpl<T>
     pdata->getParticleSys()->resize(head.dim);  // ensure that parent particle system has same size
     pdata->resize(head.dim);
 
+    const Vec3i curGridSize = pdata->getParent()->getGridSize();
+    const Vec3i headGridSize(head.dimX, head.dimY, head.dimZ);
+#  if BLENDER
+    // Correct grid size is only a soft requirement in Blender
+    if (headGridSize != curGridSize) {
+      debMsg("readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize, 1);
+      return 0;
+    }
+#  else
+    assertMsg(headGridSize == curGridSize,
+              "readPdataUni: Grid dim doesn't match, " << headGridSize << " vs " << curGridSize);
+#  endif
+
     assertMsg(head.dim == pdata->size(), "pdata size doesn't match");
 #  if FLOATINGPOINT_PRECISION != 1
     ParticleDataImpl<T> temp(pdata->getParent());
diff --git a/extern/mantaflow/preprocessed/fileio/ioutil.cpp b/extern/mantaflow/preprocessed/fileio/ioutil.cpp
index 2c7ac7be00e..2bac34c2a65 100644
--- a/extern/mantaflow/preprocessed/fileio/ioutil.cpp
+++ b/extern/mantaflow/preprocessed/fileio/ioutil.cpp
@@ -90,6 +90,13 @@ template<> void convertFrom(openvdb::Vec3s &in, Vec3 *out)
   (*out).z = in.z();
 }
 
+template<> void convertFrom(openvdb::Vec3i &in, Vec3i *out)
+{
+  (*out).x = in.x();
+  (*out).y = in.y();
+  (*out).z = in.z();
+}
+
 // Convert to OpenVDB value from Manta value.
 template<class S, class T> void convertTo(S *out, T &in)
 {
diff --git a/extern/mantaflow/preprocessed/fileio/iovdb.cpp b/extern/mantaflow/preprocessed/fileio/iovdb.cpp
index 3641577beca..cc2d0aa508d 100644
--- a/extern/mantaflow/preprocessed/fileio/iovdb.cpp
+++ b/extern/mantaflow/preprocessed/fileio/iovdb.cpp
@@ -38,6 +38,11 @@
 #define POSITION_NAME "P"
 #define FLAG_NAME "U"
 
+#define META_BASE_RES "file_base_resolution"
+#define META_VOXEL_SIZE "file_voxel_size"
+#define META_BBOX_MAX "file_bbox_max"
+#define META_BBOX_MIN "file_bbox_min"
+
 using namespace std;
 
 namespace Manta {
@@ -388,7 +393,8 @@ int writeObjectsVDB(const string &filename,
                     int compression,
                     int precision,
                     float clip,
-                    const Grid<Real> *clipGrid)
+                    const Grid<Real> *clipGrid,
+                    const bool meta)
 {
   openvdb::initialize();
   openvdb::io::File file(filename);
@@ -489,6 +495,16 @@ int writeObjectsVDB(const string &filename,
     // Set additional grid attributes, e.g. name, grid class, compression level, etc.
     if (vdbGrid) {
       setGridOptions<openvdb::GridBase>(vdbGrid, objectName, gClass, voxelSize, precision);
+
+      // Optional metadata: Save additional simulation information per vdb object
+      if (meta) {
+        const Vec3i size = object->getParent()->getGridSize();
+        // The (dense) resolution of this grid
+        vdbGrid->insertMeta(META_BASE_RES,
+                            openvdb::Vec3IMetadata(openvdb::Vec3i(size.x, size.y, size.z)));
+        // Length of one voxel side
+        vdbGrid->insertMeta(META_VOXEL_SIZE, openvdb::FloatMetadata(voxelSize));
+      }
     }
   }
 
@@ -533,6 +549,44 @@ int writeObjectsVDB(const string &filename,
   return 1;
 }
 
+static void clearAll(std::vector<PbClass *> *objects, std::vector<ParticleDataBase *> pdbBuffer)
+{
+  // Clear all data loaded into manta objects (e.g. during IO error)
+  for (std::vector<PbClass *>::iterator iter = objects->begin(); iter != objects->end(); ++iter) {
+    if (GridBase *mantaGrid = dynamic_cast<GridBase *>(*iter)) {
+      if (mantaGrid->getType() & GridBase::TypeInt) {
+        Grid<int> *mantaIntGrid = (Grid<int> *)mantaGrid;
+        mantaIntGrid->clear();
+      }
+      else if (mantaGrid->getType() & GridBase::TypeReal) {
+        Grid<Real> *mantaRealGrid = (Grid<Real> *)mantaGrid;
+        mantaRealGrid->clear();
+      }
+      else if (mantaGrid->getType() & GridBase::TypeVec3) {
+        Grid<Vec3> *mantaVec3Grid = (Grid<Vec3> *)mantaGrid;
+        mantaVec3Grid->clear();
+      }
+    }
+    else if (BasicParticleSystem *mantaPP = dynamic_cast<BasicParticleSystem *>(*iter)) {
+      mantaPP->clear();
+    }
+  }
+  for (ParticleDataBase *pdb : pdbBuffer) {
+    if (pdb->getType() == ParticleDataBase::TypeInt) {
+      ParticleDataImpl<int> *mantaPDataInt = (ParticleDataImpl<int> *)pdb;
+      mantaPDataInt->clear();
+    }
+    else if (pdb->getType() == ParticleDataBase::TypeReal) {
+      ParticleDataImpl<Real> *mantaPDataReal = (ParticleDataImpl<Real> *)pdb;
+      mantaPDataReal->clear();
+    }
+    else if (pdb->getType() == ParticleDataBase::TypeVec3) {
+      ParticleDataImpl<Vec3> *mantaPDataVec3 = (ParticleDataImpl<Vec3> *)pdb;
+      mantaPDataVec3->clear();
+    }
+  }
+}
+
 int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, float worldSize)
 {
 
@@ -561,6 +615,9 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
   // A buffer to store a handle to pData objects. These will be read alongside a particle system.
   std::vector<ParticleDataBase *> pdbBuffer;
 
+  // Count how many objects could not be read correctly
+  int readFailure = 0;
+
   for (std::vector<PbClass *>::iterator iter = objects->begin(); iter != objects->end(); ++iter) {
 
     if (gridsVDB.empty()) {
@@ -568,11 +625,12 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
     }
     // If there is just one grid in this file, load it regardless of name match (to vdb caches per
     // grid).
-    bool onlyGrid = (gridsVDB.size() == 1);
+    const bool onlyGrid = (gridsVDB.size() == 1);
 
     PbClass *object = dynamic_cast<PbClass *>(*iter);
     const Real dx = object->getParent()->getDx();
-    const Real voxelSize = worldSize * dx;
+    const Vec3i origRes = object->getParent()->getGridSize();
+    Real voxelSize = worldSize * dx;
 
     // Particle data objects are treated separately - buffered and inserted when reading the
     // particle system
@@ -596,6 +654,81 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
       if (!nameMatch && !onlyGrid) {
         continue;
       }
+
+      // Metadata: If present in the file, meta data will be parsed into these fields
+      Real metaVoxelSize(0);
+      Vec3i metaRes(0), metaBBoxMax(0), metaBBoxMin(0);
+
+      // Loop to load all meta data that we care about
+      for (openvdb::MetaMap::MetaIterator iter = vdbGrid->beginMeta(); iter != vdbGrid->endMeta();
+           ++iter) {
+        const std::string &name = iter->first;
+        const openvdb::Metadata::Ptr value = iter->second;
+        if (name.compare(META_BASE_RES) == 0) {
+          openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
+          convertFrom(tmp, &metaRes);
+        }
+        else if (name.compare(META_VOXEL_SIZE) == 0) {
+          float tmp = static_cast<openvdb::FloatMetadata &>(*value).value();
+          convertFrom(tmp, &metaVoxelSize);
+
+          voxelSize = metaVoxelSize;  // Make sure to update voxel size variable (used in
+                                      // pointgrid's importVDB())
+          if (worldSize != 1.0)
+            debMsg(
+                "readObjectsVDB: Found voxel size in meta data. worldSize parameter will be "
+                "ignored!",
+                1);
+        }
+        else if (name.compare(META_BBOX_MAX) == 0) {
+          openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
+          convertFrom(tmp, &metaBBoxMax);
+        }
+        else if (name.compare(META_BBOX_MIN) == 0) {
+          openvdb::Vec3i tmp = static_cast<openvdb::Vec3IMetadata &>(*value).value();
+          convertFrom(tmp, &metaBBoxMin);
+        }
+        else {
+          debMsg("readObjectsVDB: Skipping unknown meta information '" << name << "'", 1);
+        }
+      }
+
+      // Compare metadata with allocated grid setup. This prevents invalid index access.
+      if (notZero(metaRes) && metaRes != origRes) {
+        debMsg("readObjectsVDB Warning: Grid '" << vdbGrid->getName()
+                                                << "' has not been read. Meta grid res " << metaRes
+                                                << " vs " << origRes << " current grid size",
+               1);
+        readFailure++;
+        break;
+      }
+      if (notZero(metaVoxelSize) && metaVoxelSize != voxelSize) {
+        debMsg("readObjectsVDB Warning: Grid '"
+                   << vdbGrid->getName() << "' has not been read. Meta voxel size "
+                   << metaVoxelSize << " vs " << voxelSize << " current voxel size",
+               1);
+        readFailure++;
+        break;
+      }
+      if (metaBBoxMax.x > origRes.x || metaBBoxMax.y > origRes.y || metaBBoxMax.z > origRes.z) {
+        debMsg("readObjectsVDB Warning: Grid '"
+                   << vdbGrid->getName() << "' has not been read. Vdb bbox max " << metaBBoxMax
+                   << " vs " << origRes << " current grid size",
+               1);
+        readFailure++;
+        break;
+      }
+      const Vec3i origOrigin(0);
+      if (metaBBoxMin.x < origOrigin.x || metaBBoxMin.y < origOrigin.y ||
+          metaBBoxMin.z < origOrigin.z) {
+        debMsg("readObjectsVDB Warning: Grid '"
+                   << vdbGrid->getName() << "' has not been read. Vdb bbox min " << metaBBoxMin
+                   << " vs " << origOrigin << " current grid origin",
+               1);
+        readFailure++;
+        break;
+      }
+
       if (GridBase *mantaGrid = dynamic_cast<GridBase *>(*iter)) {
 
         if (mantaGrid->getType() & GridBase::TypeInt) {
@@ -655,6 +788,17 @@ int readObjectsVDB(const string &filename, std::vector<PbClass *> *objects, floa
         return 0;
       }
     }
+    // Do not continue loading objects in this loop if there was a read error
+    if (readFailure > 0) {
+      break;
+    }
+  }
+
+  if (readFailure > 0) {
+    // Clear all data that has already been loaded into simulation objects
+    clearAll(objects, pdbBuffer);
+    pdbBuffer.clear();
+    return 0;
   }
 
   // Give out a warning if pData items were present but could not be read due to missing particle
@@ -729,7 +873,8 @@ int writeObjectsVDB(const string &filename,
                     int compression,
                     int precision,
                     float clip,
-                    const Grid<Real> *clipGrid)
+                    const Grid<Real> *clipGrid,
+                    const bool meta)
 {
   errMsg("Cannot save to .vdb file. Mantaflow has not been built with OpenVDB support.");
   return 0;
diff --git a/extern/mantaflow/preprocessed/fileio/mantaio.cpp b/extern/mantaflow/preprocessed/fileio/mantaio.cpp
index fe29890ec11..ff57ab532e6 100644
--- a/extern/mantaflow/preprocessed/fileio/mantaio.cpp
+++ b/extern/mantaflow/preprocessed/fileio/mantaio.cpp
@@ -85,7 +85,8 @@ int save(const string &name,
          bool precisionHalf = true,
          int precision = PRECISION_HALF,
          float clip = 1e-4,
-         const Grid<Real> *clipGrid = nullptr)
+         const Grid<Real> *clipGrid = nullptr,
+         const bool meta = false)
 {
 
   if (!precisionHalf) {
@@ -105,7 +106,7 @@ int save(const string &name,
     return writeGridsVol(name, &objects);
   if (ext == ".vdb")
     return writeObjectsVDB(
-        name, &objects, worldSize, skipDeletedParts, compression, precision, clip, clipGrid);
+        name, &objects, worldSize, skipDeletedParts, compression, precision, clip, clipGrid, meta);
   else if (ext == ".npz")
     return writeGridsNumpy(name, &objects);
   else if (ext == ".txt")
@@ -134,6 +135,7 @@ static PyObject *_W_1(PyObject *_self, PyObject *_linargs, PyObject *_kwds)
       int precision = _args.getOpt<int>("precision", 6, PRECISION_HALF, &_lock);
       float clip = _args.getOpt<float>("clip", 7, 1e-4, &_lock);
       const Grid<Real> *clipGrid = _args.getPtrOpt<Grid<Real>>("clipGrid", 8, nullptr, &_lock);
+      const bool meta = _args.getOpt<bool>("meta", 9, false, &_lock);
       _retval = toPy(save(name,
                           objects,
                           worldSize,
@@ -142,7 +144,8 @@ static PyObject *_W_1(PyObject *_self, PyObject *_linargs, PyObject *_kwds)
                           precisionHalf,
                           precision,
                           clip,
-                          clipGrid));
+                          clipGrid,
+                          meta));
       _args.check();
     }
     pbFinalizePlugin(parent, "save", !noTiming);
diff --git a/extern/mantaflow/preprocessed/fileio/mantaio.h b/extern/mantaflow/preprocessed/fileio/mantaio.h
index 088d43556e1..26e9319e5e9 100644
--- a/extern/mantaflow/preprocessed/fileio/mantaio.h
+++ b/extern/mantaflow/preprocessed/fileio/mantaio.h
@@ -77,7 +77,8 @@ int writeObjectsVDB(const std::string &filename,
                     int compression = COMPRESSION_ZIP,
                     int precision = PRECISION_HALF,
                     float clip = 1e-4,
-                    const Grid<Real> *clipGrid = nullptr);
+                    const Grid<Real> *clipGrid = nullptr,
+                    const bool meta = false);
 int readObjectsVDB(const std::string &filename,
                    std::vector<PbClass *> *objects,
                    float scale = 1.0);
diff --git a/extern/mantaflow/preprocessed/gitinfo.h b/extern/mantaflow/preprocessed/gitinfo.h
index afc34a00d3d..ddc644db724 100644
--- a/extern/mantaflow/preprocessed/gitinfo.h
+++ b/extern/mantaflow/preprocessed/gitinfo.h
@@ -1,3 +1,3 @@
 
 
-#define MANTA_GIT_VERSION "commit e2285cb9bc492987f728123be6cfc1fe11fe73d6"
+#define MANTA_GIT_VERSION "commit 1c86d86496e7f7473c36248d12ef07bf4d9d2840"
diff --git a/extern/mantaflow/preprocessed/grid.cpp b/extern/mantaflow/preprocessed/grid.cpp
index 61672129f37..e0ea3830fae 100644
--- a/extern/mantaflow/preprocessed/grid.cpp
+++ b/extern/mantaflow/preprocessed/grid.cpp
@@ -508,8 +508,7 @@ struct CompMaxVec : public KernelBase {
 
 template<class T> Grid<T> &Grid<T>::copyFrom(const Grid<T> &a, bool copyType)
 {
-  assertMsg(a.mSize.x == mSize.x && a.mSize.y == mSize.y && a.mSize.z == mSize.z,
-            "different grid resolutions " << a.mSize << " vs " << this->mSize);
+  assertMsg(a.mSize == mSize, "different grid resolutions " << a.mSize << " vs " << this->mSize);
   memcpy(mData, a.mData, sizeof(T) * mSize.x * mSize.y * mSize.z);
   if (copyType)
     mType = a.mType;  // copy type marker
@@ -3402,8 +3401,7 @@ void PbRegister_markIsolatedFluidCell()
 void copyMACData(
     const MACGrid &source, MACGrid &target, const FlagGrid &flags, const int flag, const int bnd)
 {
-  assertMsg(source.getSize().x == target.getSize().x && source.getSize().y == target.getSize().y &&
-                source.getSize().z == target.getSize().z,
+  assertMsg(source.getSize() == target.getSize(),
             "different grid resolutions " << source.getSize() << " vs " << target.getSize());
 
   // Grid<Real> divGrid(target.getParent());
diff --git a/extern/mantaflow/preprocessed/grid.h b/extern/mantaflow/preprocessed/grid.h
index cf942a19e9a..3d6f8558b8f 100644
--- a/extern/mantaflow/preprocessed/grid.h
+++ b/extern/mantaflow/preprocessed/grid.h
@@ -596,6 +596,7 @@ template<class T> class Grid : public GridBase {
   //! set data
   inline void set(int i, int j, int k, T &val)
   {
+    DEBUG_ONLY(checkIndex(i, j, k));
     mData[index(i, j, k)] = val;
   }
 
diff --git a/extern/mantaflow/preprocessed/grid4d.cpp b/extern/mantaflow/preprocessed/grid4d.cpp
index 1a79a835854..72bd3a6fe50 100644
--- a/extern/mantaflow/preprocessed/grid4d.cpp
+++ b/extern/mantaflow/preprocessed/grid4d.cpp
@@ -491,9 +491,7 @@ template<class T> Grid4d<T> &Grid4d<T>::safeDivide(const Grid4d<T> &a)
 }
 template<class T> Grid4d<T> &Grid4d<T>::copyFrom(const Grid4d<T> &a, bool copyType)
 {
-  assertMsg(a.mSize.x == mSize.x && a.mSize.y == mSize.y && a.mSize.z == mSize.z &&
-                a.mSize.t == mSize.t,
-            "different Grid4d resolutions " << a.mSize << " vs " << this->mSize);
+  assertMsg(a.mSize == mSize, "different Grid4d resolutions " << a.mSize << " vs " << this->mSize);
   memcpy(mData, a.mData, sizeof(T) * mSize.x * mSize.y * mSize.z * mSize.t);
   if (copyType)
     mType = a.mType;  // copy type marker