From 7d59f84708fc8a5587b4155cd14162529a4c9baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Barschkis?= Date: Sun, 29 Mar 2020 21:30:55 +0200 Subject: Fluid: Optimization for liquid / secondary particle file loading Improved loading times for particles files by reading bigger chunks of data from the disk at once. --- intern/mantaflow/CMakeLists.txt | 1 + intern/mantaflow/intern/MANTA_main.cpp | 150 +++++++++++++++++++++++++-------- 2 files changed, 117 insertions(+), 34 deletions(-) (limited to 'intern') diff --git a/intern/mantaflow/CMakeLists.txt b/intern/mantaflow/CMakeLists.txt index 5080f2d8cac..73bcc017c83 100644 --- a/intern/mantaflow/CMakeLists.txt +++ b/intern/mantaflow/CMakeLists.txt @@ -50,6 +50,7 @@ set(INC_SYS ../../extern/mantaflow/helper/util ../../extern/mantaflow/helper/pwrapper ../../extern/mantaflow/preprocessed + ../guardedalloc ${PYTHON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} ) diff --git a/intern/mantaflow/intern/MANTA_main.cpp b/intern/mantaflow/intern/MANTA_main.cpp index c169d242583..2b3a282d815 100644 --- a/intern/mantaflow/intern/MANTA_main.cpp +++ b/intern/mantaflow/intern/MANTA_main.cpp @@ -21,10 +21,10 @@ * \ingroup mantaflow */ -#include #include -#include #include +#include +#include #include #if OPENVDB == 1 @@ -32,24 +32,29 @@ #endif #include "MANTA_main.h" -#include "manta.h" #include "Python.h" #include "fluid_script.h" -#include "smoke_script.h" #include "liquid_script.h" +#include "manta.h" +#include "smoke_script.h" +#include "BLI_fileops.h" #include "BLI_path_util.h" #include "BLI_utildefines.h" -#include "BLI_fileops.h" -#include "DNA_scene_types.h" -#include "DNA_modifier_types.h" #include "DNA_fluid_types.h" +#include "DNA_modifier_types.h" +#include "DNA_scene_types.h" + +#include "MEM_guardedalloc.h" std::atomic MANTA::mantaInitialized(false); std::atomic MANTA::solverID(0); int MANTA::with_debug(0); +/* Number of particles that the cache reads at once (with zlib). */ +#define PARTICLE_CHUNK 20000 + MANTA::MANTA(int *res, FluidModifierData *mmd) : mCurrentID(++solverID) { if (with_debug) @@ -1052,8 +1057,6 @@ int MANTA::updateFlipStructures(FluidModifierData *mmd, int framenr) BLI_path_join( cacheDir, sizeof(cacheDir), mmd->domain->cache_directory, FLUID_DOMAIN_DIR_DATA, nullptr); - // TODO (sebbas): Use pp_xl and pVel_xl when using upres simulation? - ss << "pp_####" << pformat; BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str()); BLI_path_frame(targetFile, framenr, 0); @@ -2703,18 +2706,17 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo std::cout << "MANTA::updateParticlesFromUni()" << std::endl; gzFile gzf; - float fbuffer[4]; int ibuffer[4]; gzf = (gzFile)BLI_gzopen(filename, "rb1"); // do some compression if (!gzf) - std::cout << "updateParticlesFromUni: unable to open file" << std::endl; + std::cerr << "updateParticlesFromUni: unable to open file" << std::endl; char ID[5] = {0, 0, 0, 0, 0}; gzread(gzf, ID, 4); if (!strcmp(ID, "PB01")) { - std::cout << "particle uni file format v01 not supported anymore" << std::endl; + std::cerr << "particle uni file format v01 not supported anymore" << std::endl; gzclose(gzf); return; } @@ -2757,48 +2759,128 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo } if (!ibuffer[0]) { // Any particles present? if (with_debug) - std::cout << "no particles present yet" << std::endl; + std::cerr << "no particles present yet" << std::endl; gzclose(gzf); return; } numParticles = ibuffer[0]; + const int numChunks = (int)(ceil((float)numParticles / PARTICLE_CHUNK)); + int todoParticles, readLen, k; + int readStart, readEnd, readBytes; + // Reading base particle system file v2 if (!strcmp(ID, "PB02")) { - dataPointer->resize(numParticles); MANTA::pData *bufferPData; - for (std::vector::iterator it = dataPointer->begin(); it != dataPointer->end(); ++it) { - gzread(gzf, fbuffer, sizeof(float) * 3 + sizeof(int)); - bufferPData = (MANTA::pData *)fbuffer; - it->pos[0] = bufferPData->pos[0]; - it->pos[1] = bufferPData->pos[1]; - it->pos[2] = bufferPData->pos[2]; - it->flag = bufferPData->flag; + todoParticles = numParticles; + bufferPData = (MANTA::pData *)MEM_malloc_arrayN( + PARTICLE_CHUNK, sizeof(pData), "fluid_particle_data"); + + dataPointer->resize(numParticles); + + for (int i = 0; i < numChunks && todoParticles > 0; ++i) { + readLen = PARTICLE_CHUNK; + if (todoParticles < PARTICLE_CHUNK) { + readLen = todoParticles; + } + + readBytes = gzread(gzf, bufferPData, readLen * sizeof(pData)); + if (!readBytes) { + if (with_debug) + std::cerr << "error while reading particle data" << std::endl; + MEM_freeN(bufferPData); + gzclose(gzf); + return; + } + + readStart = (numParticles - todoParticles); + CLAMP(readStart, 0, numParticles); + readEnd = readStart + PARTICLE_CHUNK; + CLAMP(readEnd, 0, numParticles); + + for (std::vector::size_type j = readStart, k = 0; j < readEnd; j++, k++) { + dataPointer->at(j).pos[0] = bufferPData[k].pos[0]; + dataPointer->at(j).pos[1] = bufferPData[k].pos[1]; + dataPointer->at(j).pos[2] = bufferPData[k].pos[2]; + dataPointer->at(j).flag = bufferPData[k].flag; + } + todoParticles -= PARTICLE_CHUNK; } + MEM_freeN(bufferPData); } // Reading particle data file v1 with velocities else if (!strcmp(ID, "PD01") && isVelData) { - velocityPointer->resize(numParticles); MANTA::pVel *bufferPVel; - for (std::vector::iterator it = velocityPointer->begin(); it != velocityPointer->end(); - ++it) { - gzread(gzf, fbuffer, sizeof(float) * 3); - bufferPVel = (MANTA::pVel *)fbuffer; - it->pos[0] = bufferPVel->pos[0]; - it->pos[1] = bufferPVel->pos[1]; - it->pos[2] = bufferPVel->pos[2]; + todoParticles = numParticles; + bufferPVel = (MANTA::pVel *)MEM_malloc_arrayN( + PARTICLE_CHUNK, sizeof(pVel), "fluid_particle_velocity"); + + velocityPointer->resize(numParticles); + + for (int i = 0; i < numChunks && todoParticles > 0; ++i) { + readLen = PARTICLE_CHUNK; + if (todoParticles < PARTICLE_CHUNK) { + readLen = todoParticles; + } + + readBytes = gzread(gzf, bufferPVel, readLen * sizeof(pVel)); + if (!readBytes) { + if (with_debug) + std::cerr << "error while reading particle velocities" << std::endl; + MEM_freeN(bufferPVel); + gzclose(gzf); + return; + } + + readStart = (numParticles - todoParticles); + CLAMP(readStart, 0, numParticles); + readEnd = readStart + PARTICLE_CHUNK; + CLAMP(readEnd, 0, numParticles); + + for (std::vector::size_type j = readStart, k = 0; j < readEnd; j++, k++) { + velocityPointer->at(j).pos[0] = bufferPVel[k].pos[0]; + velocityPointer->at(j).pos[1] = bufferPVel[k].pos[1]; + velocityPointer->at(j).pos[2] = bufferPVel[k].pos[2]; + } + todoParticles -= PARTICLE_CHUNK; } + MEM_freeN(bufferPVel); } // Reading particle data file v1 with lifetime else if (!strcmp(ID, "PD01")) { - lifePointer->resize(numParticles); float *bufferPLife; - for (std::vector::iterator it = lifePointer->begin(); it != lifePointer->end(); ++it) { - gzread(gzf, fbuffer, sizeof(float)); - bufferPLife = (float *)fbuffer; - *it = *bufferPLife; + todoParticles = numParticles; + bufferPLife = (float *)MEM_malloc_arrayN(PARTICLE_CHUNK, sizeof(float), "fluid_particle_life"); + + lifePointer->resize(numParticles); + + for (int i = 0; i < numChunks && todoParticles > 0; ++i) { + readLen = PARTICLE_CHUNK; + if (todoParticles < PARTICLE_CHUNK) { + readLen = todoParticles; + } + + readBytes = gzread(gzf, bufferPLife, readLen * sizeof(float)); + if (!readBytes) { + if (with_debug) + std::cerr << "error while reading particle life" << std::endl; + MEM_freeN(bufferPLife); + gzclose(gzf); + return; + } + + readStart = (numParticles - todoParticles); + CLAMP(readStart, 0, numParticles); + readEnd = readStart + PARTICLE_CHUNK; + CLAMP(readEnd, 0, numParticles); + + for (std::vector::size_type j = readStart, k = 0; j < readEnd; j++, k++) { + lifePointer->at(j) = bufferPLife[k]; + } + todoParticles -= PARTICLE_CHUNK; } + MEM_freeN(bufferPLife); } gzclose(gzf); -- cgit v1.2.3