Refactor low-level blendfile reading into separate files

Instead of handling mmap, compression etc. all directly in readfile.c, refactor
the code to use a generic FileReader.
This makes it easier to add new compression methods or similar, and allows to
reuse the logic in other places (e.g. thumbnail reading).

Reviewed By: campbellbarton, brecht, mont29

Differential Revision: https://developer.blender.org/D5799

4 files changed, 340 insertions, 0 deletions

diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index ac034d2b5cd..6fc2222241b 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
 
 #undef CHUNK
 
+bool BLI_file_magic_is_gzip(const char header[4])
+{
+  /* GZIP itself starts with the magic bytes 0x1f 0x8b.
+   * The third byte indicates the compression method, which is 0x08 for DEFLATE. */
+  return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
+}
+
 /**
  * Returns true if the file with the specified name can be written.
  * This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c
new file mode 100644
index 00000000000..3a833871e27
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_file.c
@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#ifndef WIN32
+#  include <unistd.h> /* for read close */
+#else
+#  include "BLI_winstuff.h"
+#  include "winsock2.h"
+#  include <io.h> /* for open close read */
+#endif
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  int filedes;
+} RawFileReader;
+
+static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  ssize_t readsize = read(rawfile->filedes, buffer, size);
+
+  if (readsize >= 0) {
+    rawfile->reader.offset += readsize;
+  }
+
+  return readsize;
+}
+
+static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
+  return rawfile->reader.offset;
+}
+
+static void file_close(FileReader *reader)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  close(rawfile->filedes);
+  MEM_freeN(rawfile);
+}
+
+FileReader *BLI_filereader_new_file(int filedes)
+{
+  RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
+
+  rawfile->filedes = filedes;
+
+  rawfile->reader.read = file_read;
+  rawfile->reader.seek = file_seek;
+  rawfile->reader.close = file_close;
+
+  return (FileReader *)rawfile;
+}
diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c
new file mode 100644
index 00000000000..72eb153a8b9
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_gzip.c
@@ -0,0 +1,108 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <zlib.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  FileReader *base;
+
+  z_stream strm;
+
+  void *in_buf;
+  size_t in_size;
+} GzipReader;
+
+static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  gzip->strm.avail_out = size;
+  gzip->strm.next_out = buffer;
+
+  while (gzip->strm.avail_out > 0) {
+    if (gzip->strm.avail_in == 0) {
+      /* Ran out of buffered input data, read some more. */
+      size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
+
+      if (readsize > 0) {
+        /* We got some data, so mark the buffer as refilled. */
+        gzip->strm.avail_in = readsize;
+        gzip->strm.next_in = gzip->in_buf;
+      }
+      else {
+        /* The underlying file is EOF, so return as much as we can. */
+        break;
+      }
+    }
+
+    int ret = inflate(&gzip->strm, Z_NO_FLUSH);
+
+    if (ret != Z_OK && ret != Z_BUF_ERROR) {
+      break;
+    }
+  }
+
+  ssize_t read_len = size - gzip->strm.avail_out;
+  gzip->reader.offset += read_len;
+  return read_len;
+}
+
+static void gzip_close(FileReader *reader)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  if (inflateEnd(&gzip->strm) != Z_OK) {
+    printf("close gzip stream error\n");
+  }
+  MEM_freeN((void *)gzip->in_buf);
+
+  gzip->base->close(gzip->base);
+  MEM_freeN(gzip);
+}
+
+FileReader *BLI_filereader_new_gzip(FileReader *base)
+{
+  GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
+  gzip->base = base;
+
+  if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
+    MEM_freeN(gzip);
+    return NULL;
+  }
+
+  gzip->in_size = 256 * 2014;
+  gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
+
+  gzip->reader.read = gzip_read;
+  gzip->reader.seek = NULL;
+  gzip->reader.close = gzip_close;
+
+  return (FileReader *)gzip;
+}
diff --git a/source/blender/blenlib/intern/filereader_memory.c b/source/blender/blenlib/intern/filereader_memory.c
new file mode 100644
index 00000000000..150fed7d1cc
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_memory.c
@@ -0,0 +1,145 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+#include "BLI_mmap.h"
+
+#include "MEM_guardedalloc.h"
+
+/* This file implements both memory-backed and memory-mapped-file-backed reading. */
+typedef struct {
+  FileReader reader;
+
+  const char *data;
+  BLI_mmap_file *mmap;
+  size_t length;
+} MemoryReader;
+
+static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  memcpy(buffer, mem->data + mem->reader.offset, readsize);
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  off64_t new_pos;
+  if (whence == SEEK_CUR) {
+    new_pos = mem->reader.offset + offset;
+  }
+  else if (whence == SEEK_SET) {
+    new_pos = offset;
+  }
+  else if (whence == SEEK_END) {
+    new_pos = mem->length + offset;
+  }
+  else {
+    return -1;
+  }
+
+  if (new_pos < 0 || new_pos > mem->length) {
+    return -1;
+  }
+
+  mem->reader.offset = new_pos;
+  return mem->reader.offset;
+}
+
+static void memory_close_raw(FileReader *reader)
+{
+  MEM_freeN(reader);
+}
+
+FileReader *BLI_filereader_new_memory(const void *data, size_t len)
+{
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->data = (const char *)data;
+  mem->length = len;
+
+  mem->reader.read = memory_read_raw;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_raw;
+
+  return (FileReader *)mem;
+}
+
+/* Memory-mapped file reading.
+ * By using `mmap()`, we can map a file so that it can be treated like normal memory,
+ * meaning that we can just read from it with `memcpy()` etc.
+ * This avoids system call overhead and can significantly speed up file loading.
+ */
+
+static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
+    return 0;
+  }
+
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static void memory_close_mmap(FileReader *reader)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+  BLI_mmap_free(mem->mmap);
+  MEM_freeN(mem);
+}
+
+FileReader *BLI_filereader_new_mmap(int filedes)
+{
+  BLI_mmap_file *mmap = BLI_mmap_open(filedes);
+  if (mmap == NULL) {
+    return NULL;
+  }
+
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->mmap = mmap;
+  mem->length = BLI_lseek(filedes, 0, SEEK_END);
+
+  mem->reader.read = memory_read_mmap;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_mmap;
+
+  return (FileReader *)mem;
+}