Use mmap() IO for reading uncompressed .blends

Instead of submitting tons of tiny IO syscalls, we can speed things up
significantly by `mmap`ing the .blend file into virtual memory and directly
accessing it.

In my local testing, this speeds up loading the Dweebs file with all its
linked files from 19sec to 10sec (on Linux).

As far as I can see, this should be supported on Linux, OSX and BSD.
For Windows, a second code path uses `CreateFileMapping` and
`MapViewOfFile` to achieve the same result.

Reviewed By: mont29, brecht

Differential Revision: https://developer.blender.org/D8246

3 files changed, 13 insertions, 15 deletions

diff --git a/source/blender/imbuf/intern/IMB_allocimbuf.h b/source/blender/imbuf/intern/IMB_allocimbuf.h
index 08aa1936a6f..c92d764a104 100644
--- a/source/blender/imbuf/intern/IMB_allocimbuf.h
+++ b/source/blender/imbuf/intern/IMB_allocimbuf.h
@@ -32,7 +32,7 @@ struct ImBuf;
 void imb_refcounter_lock_init(void);
 void imb_refcounter_lock_exit(void);
 
-#ifdef WIN32
+#ifndef WIN32
 void imb_mmap_lock_init(void);
 void imb_mmap_lock_exit(void);
 void imb_mmap_lock(void);
diff --git a/source/blender/imbuf/intern/allocimbuf.c b/source/blender/imbuf/intern/allocimbuf.c
index 8dfb3ada7d6..bfb7bc93ac7 100644
--- a/source/blender/imbuf/intern/allocimbuf.c
+++ b/source/blender/imbuf/intern/allocimbuf.c
@@ -53,7 +53,7 @@ void imb_refcounter_lock_exit(void)
   BLI_spin_end(&refcounter_spin);
 }
 
-#ifdef WIN32
+#ifndef WIN32
 static SpinLock mmap_spin;
 
 void imb_mmap_lock_init(void)
diff --git a/source/blender/imbuf/intern/readimage.c b/source/blender/imbuf/intern/readimage.c
index f0daa4543e1..50210650f05 100644
--- a/source/blender/imbuf/intern/readimage.c
+++ b/source/blender/imbuf/intern/readimage.c
@@ -23,13 +23,13 @@
  */
 
 #ifdef _WIN32
-#  include "mmap_win.h"
 #  include <io.h>
 #  include <stddef.h>
 #  include <sys/types.h>
 #endif
 
 #include "BLI_fileops.h"
+#include "BLI_mmap.h"
 #include "BLI_path_util.h"
 #include "BLI_string.h"
 #include "BLI_utildefines.h"
@@ -186,20 +186,19 @@ ImBuf *IMB_loadifffile(
   size = BLI_file_descriptor_size(file);
 
   imb_mmap_lock();
-  mem = mmap(NULL, size, PROT_READ, MAP_SHARED, file, 0);
+  BLI_mmap_file *mmap_file = BLI_mmap_open(file);
   imb_mmap_unlock();
-
-  if (mem == (unsigned char *)-1) {
+  if (mmap_file == NULL) {
     fprintf(stderr, "%s: couldn't get mapping %s\n", __func__, descr);
     return NULL;
   }
 
+  mem = BLI_mmap_get_pointer(mmap_file);
+
   ibuf = IMB_ibImageFromMemory(mem, size, flags, colorspace, descr);
 
   imb_mmap_lock();
-  if (munmap(mem, size)) {
-    fprintf(stderr, "%s: couldn't unmap file %s\n", __func__, descr);
-  }
+  BLI_mmap_free(mmap_file);
   imb_mmap_unlock();
 
   return ibuf;
@@ -292,14 +291,15 @@ static void imb_loadtilefile(ImBuf *ibuf, int file, int tx, int ty, unsigned int
   size = BLI_file_descriptor_size(file);
 
   imb_mmap_lock();
-  mem = mmap(NULL, size, PROT_READ, MAP_SHARED, file, 0);
+  BLI_mmap_file *mmap_file = BLI_mmap_open(file);
   imb_mmap_unlock();
-
-  if (mem == (unsigned char *)-1) {
+  if (mmap_file == NULL) {
     fprintf(stderr, "Couldn't get memory mapping for %s\n", ibuf->cachename);
     return;
   }
 
+  mem = BLI_mmap_get_pointer(mmap_file);
+
   const ImFileType *type = IMB_file_type_from_ibuf(ibuf);
   if (type != NULL) {
     if (type->load_tile != NULL) {
@@ -308,9 +308,7 @@ static void imb_loadtilefile(ImBuf *ibuf, int file, int tx, int ty, unsigned int
   }
 
   imb_mmap_lock();
-  if (munmap(mem, size)) {
-    fprintf(stderr, "Couldn't unmap memory for %s.\n", ibuf->cachename);
-  }
+  BLI_mmap_free(mmap_file);
   imb_mmap_unlock();
 }