From ef502127ddc0f5a06e76cb2f634655b8e9c8f740 Mon Sep 17 00:00:00 2001
From: Germano Cavalcante <germano.costa@ig.com.br>
Date: Fri, 20 Aug 2021 08:40:13 -0300
Subject: Fix T90795: Moving keys in Grease Pencil Dopesheet crashes Blender

`td->loc` is referenced but not initialized.
---
 source/blender/editors/transform/transform_convert_action.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/source/blender/editors/transform/transform_convert_action.c b/source/blender/editors/transform/transform_convert_action.c
index 8a3b254be5c..a5565b5fb88 100644
--- a/source/blender/editors/transform/transform_convert_action.c
+++ b/source/blender/editors/transform/transform_convert_action.c
@@ -241,16 +241,15 @@ static int GPLayerToTransData(TransData *td,
   for (gpf = gpl->frames.first; gpf; gpf = gpf->next) {
     if (is_prop_edit || (gpf->flag & GP_FRAME_SELECT)) {
       if (FrameOnMouseSide(side, (float)gpf->framenum, cfra)) {
-        /* memory is calloc'ed, so that should zero everything nicely for us */
-        td->val = &tfd->val;
-        td->ival = (float)gpf->framenum;
+        tfd->val = (float)gpf->framenum;
+        tfd->sdata = &gpf->framenum;
+
+        td->val = td->loc = &tfd->val; /* XXX: It's not a 3d array. */
+        td->ival = td->iloc[0] = (float)gpf->framenum;
 
         td->center[0] = td->ival;
         td->center[1] = ypos;
 
-        tfd->val = (float)gpf->framenum;
-        tfd->sdata = &gpf->framenum;
-
         /* Advance `td` now. */
         td++;
         tfd++;
-- 
cgit v1.2.3


From accf3045be40433045c197cfcdbcbc32a7724403 Mon Sep 17 00:00:00 2001
From: Germano Cavalcante <germano.costa@ig.com.br>
Date: Fri, 20 Aug 2021 09:05:52 -0300
Subject: Fix memory leak while processing mouse event

Assignment missed.
---
 intern/ghost/intern/GHOST_SystemWin32.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/intern/ghost/intern/GHOST_SystemWin32.cpp b/intern/ghost/intern/GHOST_SystemWin32.cpp
index 347067eae50..f44107ee000 100644
--- a/intern/ghost/intern/GHOST_SystemWin32.cpp
+++ b/intern/ghost/intern/GHOST_SystemWin32.cpp
@@ -1741,7 +1741,7 @@ LRESULT WINAPI GHOST_SystemWin32::s_wndProc(HWND hwnd, UINT msg, WPARAM wParam,
         case WM_MOUSELEAVE: {
           window->m_mousePresent = false;
           if (window->getTabletData().Active == GHOST_kTabletModeNone) {
-            processCursorEvent(window);
+            event = processCursorEvent(window);
           }
           GHOST_Wintab *wt = window->getWintab();
           if (wt) {
-- 
cgit v1.2.3


From 9bfc47c9334bc4fbecbe7871fff9af4cc46c8832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?K=C3=A9vin=20Dietrich?= <kevin.dietrich@mailoo.org>
Date: Fri, 20 Aug 2021 14:29:05 +0200
Subject: Alembic Procedural: basic cache control settings

This adds a setting to enable data caching, and another one to set the
maximum cache size in megabytes.

When caching is enabled we load the data for the entire animation in
memory, as we already do, however, if the data exceeds the memory limit,
render is aborted.

When caching is disabled, we simply load the data for the current frame
in memory.

Ref D10197

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D11163
---
 intern/cycles/blender/blender_object.cpp           |  3 +
 intern/cycles/render/alembic.cpp                   | 70 ++++++++++++++++++++++
 intern/cycles/render/alembic.h                     | 30 ++++++++++
 intern/cycles/render/alembic_read.cpp              | 16 ++++-
 .../editors/interface/interface_templates.c        | 15 +++++
 source/blender/makesdna/DNA_cachefile_defaults.h   |  2 +
 source/blender/makesdna/DNA_cachefile_types.h      | 10 +++-
 source/blender/makesrna/intern/rna_cachefile.c     | 17 ++++++
 8 files changed, 159 insertions(+), 4 deletions(-)

diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index 2dbebac4cc3..a7eae421b55 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -523,6 +523,9 @@ void BlenderSync::sync_procedural(BL::Object &b_ob,
 
   procedural->set_scale(cache_file.scale());
 
+  procedural->set_use_prefetch(cache_file.use_prefetch());
+  procedural->set_prefetch_cache_size(cache_file.prefetch_cache_size());
+
   /* create or update existing AlembicObjects */
   ustring object_path = ustring(b_mesh_cache.object_path());
 
diff --git a/intern/cycles/render/alembic.cpp b/intern/cycles/render/alembic.cpp
index 81f47256739..69bc0712674 100644
--- a/intern/cycles/render/alembic.cpp
+++ b/intern/cycles/render/alembic.cpp
@@ -25,6 +25,7 @@
 #include "render/shader.h"
 
 #include "util/util_foreach.h"
+#include "util/util_logging.h"
 #include "util/util_progress.h"
 #include "util/util_transform.h"
 #include "util/util_vector.h"
@@ -211,6 +212,35 @@ void CachedData::set_time_sampling(TimeSampling time_sampling)
   }
 }
 
+size_t CachedData::memory_used() const
+{
+  size_t mem_used = 0;
+
+  mem_used += curve_first_key.memory_used();
+  mem_used += curve_keys.memory_used();
+  mem_used += curve_radius.memory_used();
+  mem_used += curve_shader.memory_used();
+  mem_used += num_ngons.memory_used();
+  mem_used += shader.memory_used();
+  mem_used += subd_creases_edge.memory_used();
+  mem_used += subd_creases_weight.memory_used();
+  mem_used += subd_face_corners.memory_used();
+  mem_used += subd_num_corners.memory_used();
+  mem_used += subd_ptex_offset.memory_used();
+  mem_used += subd_smooth.memory_used();
+  mem_used += subd_start_corner.memory_used();
+  mem_used += transforms.memory_used();
+  mem_used += triangles.memory_used();
+  mem_used += uv_loops.memory_used();
+  mem_used += vertices.memory_used();
+
+  for (const CachedAttribute &attr : attributes) {
+    mem_used += attr.data.memory_used();
+  }
+
+  return mem_used;
+}
+
 static M44d convert_yup_zup(const M44d &mtx, float scale_mult)
 {
   V3d scale, shear, rotation, translation;
@@ -706,6 +736,9 @@ NODE_DEFINE(AlembicProcedural)
 
   SOCKET_NODE_ARRAY(objects, "Objects", AlembicObject::get_node_type());
 
+  SOCKET_BOOLEAN(use_prefetch, "Use Prefetch", true);
+  SOCKET_INT(prefetch_cache_size, "Prefetch Cache Size", 4096);
+
   return type;
 }
 
@@ -823,6 +856,30 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress)
     }
   }
 
+  if (use_prefetch_is_modified()) {
+    if (!use_prefetch) {
+      for (Node *node : objects) {
+        AlembicObject *object = static_cast<AlembicObject *>(node);
+        object->clear_cache();
+      }
+    }
+  }
+
+  if (prefetch_cache_size_is_modified()) {
+    /* Check whether the current memory usage fits in the new requested size,
+     * abort the render if it is any higher. */
+    size_t memory_used = 0ul;
+    for (Node *node : objects) {
+      AlembicObject *object = static_cast<AlembicObject *>(node);
+      memory_used += object->get_cached_data().memory_used();
+    }
+
+    if (memory_used > get_prefetch_cache_size_in_bytes()) {
+      progress.set_error("Error: Alembic Procedural memory limit reached");
+      return;
+    }
+  }
+
   build_caches(progress);
 
   foreach (Node *node, objects) {
@@ -1300,6 +1357,8 @@ void AlembicProcedural::walk_hierarchy(
 
 void AlembicProcedural::build_caches(Progress &progress)
 {
+  size_t memory_used = 0;
+
   for (Node *node : objects) {
     AlembicObject *object = static_cast<AlembicObject *>(node);
 
@@ -1353,7 +1412,18 @@ void AlembicProcedural::build_caches(Progress &progress)
     if (scale_is_modified() || object->get_cached_data().transforms.size() == 0) {
       object->setup_transform_cache(object->get_cached_data(), scale);
     }
+
+    memory_used += object->get_cached_data().memory_used();
+
+    if (use_prefetch) {
+      if (memory_used > get_prefetch_cache_size_in_bytes()) {
+        progress.set_error("Error: Alembic Procedural memory limit reached");
+        return;
+      }
+    }
   }
+
+  VLOG(1) << "AlembicProcedural memory usage : " << string_human_readable_size(memory_used);
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/alembic.h b/intern/cycles/render/alembic.h
index 9c58af720f6..8e166a5ab04 100644
--- a/intern/cycles/render/alembic.h
+++ b/intern/cycles/render/alembic.h
@@ -272,6 +272,21 @@ template<typename T> class DataStore {
     node->set(*socket, value);
   }
 
+  size_t memory_used() const
+  {
+    if constexpr (is_array<T>::value) {
+      size_t mem_used = 0;
+
+      for (const T &array : data) {
+        mem_used += array.size() * sizeof(array[0]);
+      }
+
+      return mem_used;
+    }
+
+    return data.size() * sizeof(T);
+  }
+
  private:
   const TimeIndexPair &get_index_for_time(double time) const
   {
@@ -332,6 +347,8 @@ struct CachedData {
   void invalidate_last_loaded_time(bool attributes_only = false);
 
   void set_time_sampling(Alembic::AbcCoreAbstract::TimeSampling time_sampling);
+
+  size_t memory_used() const;
 };
 
 /* Representation of an Alembic object for the AlembicProcedural.
@@ -482,6 +499,13 @@ class AlembicProcedural : public Procedural {
    * software. */
   NODE_SOCKET_API(float, scale)
 
+  /* Cache controls */
+  NODE_SOCKET_API(bool, use_prefetch)
+
+  /* Memory limit for the cache, if the data does not fit within this limit, rendering is aborted.
+   */
+  NODE_SOCKET_API(int, prefetch_cache_size)
+
   AlembicProcedural();
   ~AlembicProcedural();
 
@@ -531,6 +555,12 @@ class AlembicProcedural : public Procedural {
   void read_subd(AlembicObject *abc_object, Alembic::AbcGeom::Abc::chrono_t frame_time);
 
   void build_caches(Progress &progress);
+
+  size_t get_prefetch_cache_size_in_bytes() const
+  {
+    /* prefetch_cache_size is in megabytes, so convert to bytes. */
+    return static_cast<size_t>(prefetch_cache_size) * 1024 * 1024;
+  }
 };
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/alembic_read.cpp b/intern/cycles/render/alembic_read.cpp
index c53ec668938..b105af63b44 100644
--- a/intern/cycles/render/alembic_read.cpp
+++ b/intern/cycles/render/alembic_read.cpp
@@ -44,9 +44,19 @@ static set<chrono_t> get_relevant_sample_times(AlembicProcedural *proc,
     return result;
   }
 
-  // load the data for the entire animation
-  const double start_frame = static_cast<double>(proc->get_start_frame());
-  const double end_frame = static_cast<double>(proc->get_end_frame());
+  double start_frame;
+  double end_frame;
+
+  if (proc->get_use_prefetch()) {
+    // load the data for the entire animation
+    start_frame = static_cast<double>(proc->get_start_frame());
+    end_frame = static_cast<double>(proc->get_end_frame());
+  }
+  else {
+    // load the data for the current frame
+    start_frame = static_cast<double>(proc->get_frame());
+    end_frame = start_frame;
+  }
 
   const double frame_rate = static_cast<double>(proc->get_frame_rate());
   const double start_time = start_frame / frame_rate;
diff --git a/source/blender/editors/interface/interface_templates.c b/source/blender/editors/interface/interface_templates.c
index 351b73c320b..cdb35d19855 100644
--- a/source/blender/editors/interface/interface_templates.c
+++ b/source/blender/editors/interface/interface_templates.c
@@ -6489,6 +6489,17 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiLayoutSetActive(row, engine_supports_procedural);
   uiItemR(row, &fileptr, "use_render_procedural", 0, NULL, ICON_NONE);
 
+  const bool use_render_procedural = RNA_boolean_get(&fileptr, "use_render_procedural");
+  const bool use_prefetch = RNA_boolean_get(&fileptr, "use_prefetch");
+
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(row, use_render_procedural);
+  uiItemR(row, &fileptr, "use_prefetch", 0, NULL, ICON_NONE);
+
+  sub = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(sub, use_prefetch && use_render_procedural);
+  uiItemR(sub, &fileptr, "prefetch_cache_size", 0, NULL, ICON_NONE);
+
   row = uiLayoutRowWithHeading(layout, true, IFACE_("Override Frame"));
   sub = uiLayoutRow(row, true);
   uiLayoutSetPropDecorate(sub, false);
@@ -6510,6 +6521,10 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiItemR(layout, &fileptr, "velocity_name", 0, NULL, ICON_NONE);
   uiItemR(layout, &fileptr, "velocity_unit", 0, NULL, ICON_NONE);
 
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetActive(row, engine_supports_procedural && use_render_procedural);
+  uiItemR(row, &fileptr, "default_radius", 0, NULL, ICON_NONE);
+
   /* TODO: unused for now, so no need to expose. */
 #if 0
   row = uiLayoutRow(layout, false);
diff --git a/source/blender/makesdna/DNA_cachefile_defaults.h b/source/blender/makesdna/DNA_cachefile_defaults.h
index 521b72567d4..74fbe5012ab 100644
--- a/source/blender/makesdna/DNA_cachefile_defaults.h
+++ b/source/blender/makesdna/DNA_cachefile_defaults.h
@@ -40,6 +40,8 @@
     .handle = NULL, \
     .handle_filepath[0] = '\0', \
     .handle_readers = NULL, \
+    .use_prefetch = 1, \
+    .prefetch_cache_size = 4096, \
   }
 
 /** \} */
diff --git a/source/blender/makesdna/DNA_cachefile_types.h b/source/blender/makesdna/DNA_cachefile_types.h
index ae4ade49be1..0f4c53a6e7e 100644
--- a/source/blender/makesdna/DNA_cachefile_types.h
+++ b/source/blender/makesdna/DNA_cachefile_types.h
@@ -101,7 +101,15 @@ typedef struct CacheFile {
    */
   char use_render_procedural;
 
-  char _pad1[7];
+  char _pad1[3];
+
+  /** Enable data prefetching when using the Cycles Procedural. */
+  char use_prefetch;
+
+  /** Size in megabytes for the prefetch cache used by the Cycles Procedural. */
+  int prefetch_cache_size;
+
+  char _pad2[7];
 
   char velocity_unit;
   /* Name of the velocity property in the archive. */
diff --git a/source/blender/makesrna/intern/rna_cachefile.c b/source/blender/makesrna/intern/rna_cachefile.c
index cb0a490417d..74d924b8937 100644
--- a/source/blender/makesrna/intern/rna_cachefile.c
+++ b/source/blender/makesrna/intern/rna_cachefile.c
@@ -150,6 +150,23 @@ static void rna_def_cachefile(BlenderRNA *brna)
                            "determine which file to use in a file sequence");
   RNA_def_property_update(prop, 0, "rna_CacheFile_update");
 
+  /* ----------------- Cache controls ----------------- */
+
+  prop = RNA_def_property(srna, "use_prefetch", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_ui_text(
+      prop,
+      "Use Prefetch",
+      "When enabled, the Cycles Procedural will preload animation data for faster updates");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
+  prop = RNA_def_property(srna, "prefetch_cache_size", PROP_INT, PROP_UNSIGNED);
+  RNA_def_property_ui_text(
+      prop,
+      "Prefetch Cache Size",
+      "Memory usage limit in megabytes for the Cycles Procedural cache, if the data does not "
+      "fit within the limit, rendering is aborted");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
   /* ----------------- Axis Conversion ----------------- */
 
   prop = RNA_def_property(srna, "forward_axis", PROP_ENUM, PROP_NONE);
-- 
cgit v1.2.3


From 6a404bc633586a96d635e2e3bf2e207f87ab3785 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?K=C3=A9vin=20Dietrich?= <kevin.dietrich@mailoo.org>
Date: Fri, 20 Aug 2021 14:48:47 +0200
Subject: Cleanup, remove extra code from previous commit

This got accidentally introduced while revising dependencies between
patches for this feature, did not notice until it was too late.
---
 source/blender/editors/interface/interface_templates.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/source/blender/editors/interface/interface_templates.c b/source/blender/editors/interface/interface_templates.c
index cdb35d19855..0e5a6a79137 100644
--- a/source/blender/editors/interface/interface_templates.c
+++ b/source/blender/editors/interface/interface_templates.c
@@ -6521,10 +6521,6 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiItemR(layout, &fileptr, "velocity_name", 0, NULL, ICON_NONE);
   uiItemR(layout, &fileptr, "velocity_unit", 0, NULL, ICON_NONE);
 
-  row = uiLayoutRow(layout, false);
-  uiLayoutSetActive(row, engine_supports_procedural && use_render_procedural);
-  uiItemR(row, &fileptr, "default_radius", 0, NULL, ICON_NONE);
-
   /* TODO: unused for now, so no need to expose. */
 #if 0
   row = uiLayoutRow(layout, false);
-- 
cgit v1.2.3


From 1b5f17b86771faa87d36ff12183aa6dbb5c1e9df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?K=C3=A9vin=20Dietrich?= <kevin.dietrich@mailoo.org>
Date: Fri, 20 Aug 2021 15:00:58 +0200
Subject: Cleanup, use BKE_scene_uses_cycles_experimental_features

---
 source/blender/modifiers/intern/MOD_subsurf.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/source/blender/modifiers/intern/MOD_subsurf.c b/source/blender/modifiers/intern/MOD_subsurf.c
index ce427281db3..db0b769684e 100644
--- a/source/blender/modifiers/intern/MOD_subsurf.c
+++ b/source/blender/modifiers/intern/MOD_subsurf.c
@@ -358,13 +358,8 @@ static bool get_show_adaptive_options(const bContext *C, Panel *panel)
 
   /* Don't show adaptive options if the cycles experimental feature set is disabled. */
   Scene *scene = CTX_data_scene(C);
-  PointerRNA scene_ptr;
-  RNA_id_pointer_create(&scene->id, &scene_ptr);
-  if (BKE_scene_uses_cycles(scene)) {
-    PointerRNA cycles_ptr = RNA_pointer_get(&scene_ptr, "cycles");
-    if (RNA_enum_get(&cycles_ptr, "feature_set") != 1) { /* EXPERIMENTAL */
-      return false;
-    }
+  if (!BKE_scene_uses_cycles_experimental_features(scene)) {
+    return false;
   }
 
   return true;
-- 
cgit v1.2.3


From cd8d9383e74a798e75ebced95cdd6d8de9f02369 Mon Sep 17 00:00:00 2001
From: Alaska <Alaska>
Date: Fri, 20 Aug 2021 17:43:15 +0200
Subject: Fix T90804: small grammatical error in noise threshold description

Differential Revision: https://developer.blender.org/D12277
---
 intern/cycles/blender/addon/properties.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/intern/cycles/blender/addon/properties.py b/intern/cycles/blender/addon/properties.py
index ac0aca57028..0c3af3fabeb 100644
--- a/intern/cycles/blender/addon/properties.py
+++ b/intern/cycles/blender/addon/properties.py
@@ -408,7 +408,7 @@ class CyclesRenderSettings(bpy.types.PropertyGroup):
 
     adaptive_threshold: FloatProperty(
         name="Adaptive Sampling Threshold",
-        description="Noise level step to stop sampling at, lower values reduce noise the cost of render time. Zero for automatic setting based on number of AA samples",
+        description="Noise level step to stop sampling at, lower values reduce noise at the cost of render time. Zero for automatic setting based on number of AA samples",
         min=0.0, max=1.0,
         default=0.0,
         precision=4,
-- 
cgit v1.2.3


From 86622467c573e0703eea2554ebb0da5c67cbfb8c Mon Sep 17 00:00:00 2001
From: Aaron Carlisle <carlisle.b3d@gmail.com>
Date: Fri, 20 Aug 2021 13:46:38 -0400
Subject: DocPy: Update Dependancies

Updates sphinx and the theme to the latest version along with any of their dependencies.

Note that we will be sticking to sphinx 4.1.1 until sphinx 4.2 for the same reasons listed in:
https://developer.blender.org/rBM8334
---
 doc/python_api/requirements.txt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/python_api/requirements.txt b/doc/python_api/requirements.txt
index a854d2a267b..b5a9d15bf7b 100644
--- a/doc/python_api/requirements.txt
+++ b/doc/python_api/requirements.txt
@@ -1,13 +1,13 @@
-sphinx==3.5.4
+sphinx==4.1.1
 
 # Sphinx dependencies that are important
-Jinja2==2.11.3
-Pygments==2.9.0
-docutils==0.16
+Jinja2==3.0.1
+Pygments==2.10.0
+docutils==0.17.1
 snowballstemmer==2.1.0
 babel==2.9.1
-requests==2.25.1
+requests==2.26.0
 
 # Only needed to match the theme used for the official documentation.
 # Without this theme, the default theme will be used.
-sphinx_rtd_theme==0.5.2
+sphinx_rtd_theme==1.0.0rc1
-- 
cgit v1.2.3


From b6a1bf757d9f0675f0fb5f21a72ff4ee31f8d6d9 Mon Sep 17 00:00:00 2001
From: Aaron Carlisle <carlisle.b3d@gmail.com>
Date: Fri, 20 Aug 2021 14:54:46 -0400
Subject: DocPy: Cleanup missing newline resulting in wrong html generation

---
 source/blender/python/intern/bpy_app_icons.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/blender/python/intern/bpy_app_icons.c b/source/blender/python/intern/bpy_app_icons.c
index 7cca3ae4700..acd809fb8d5 100644
--- a/source/blender/python/intern/bpy_app_icons.c
+++ b/source/blender/python/intern/bpy_app_icons.c
@@ -35,7 +35,7 @@
 /* We may want to load direct from file. */
 PyDoc_STRVAR(
     bpy_app_icons_new_triangles_doc,
-    ".. function:: new_triangles(range, coords, colors)"
+    ".. function:: new_triangles(range, coords, colors)\n"
     "\n"
     "   Create a new icon from triangle geometry.\n"
     "\n"
@@ -91,7 +91,7 @@ static PyObject *bpy_app_icons_new_triangles(PyObject *UNUSED(self), PyObject *a
 }
 
 PyDoc_STRVAR(bpy_app_icons_new_triangles_from_file_doc,
-             ".. function:: new_triangles_from_file(filename)"
+             ".. function:: new_triangles_from_file(filename)\n"
              "\n"
              "   Create a new icon from triangle geometry.\n"
              "\n"
@@ -122,7 +122,7 @@ static PyObject *bpy_app_icons_new_triangles_from_file(PyObject *UNUSED(self),
 }
 
 PyDoc_STRVAR(bpy_app_icons_release_doc,
-             ".. function:: release(icon_id)"
+             ".. function:: release(icon_id)\n"
              "\n"
              "   Release the icon.\n");
 static PyObject *bpy_app_icons_release(PyObject *UNUSED(self), PyObject *args, PyObject *kw)
-- 
cgit v1.2.3


From 0d7aab2375e6bb06e89dad851550b283a1ff805c Mon Sep 17 00:00:00 2001
From: Harley Acheson <harley.acheson@gmail.com>
Date: Fri, 20 Aug 2021 17:48:42 -0700
Subject: Refactor: BLF Kerning Cache After Use

Optimization of font kerning by only caching kerning values after a
pair is encountered. Also saves unscaled values so they don't have to
be rebuilt between font size changes.

See D12274 for more details and speed comparison.

Differential Revision: https://developer.blender.org/D12274

Reviewed by Campbell Barton
---
 source/blender/blenfont/intern/blf.c               |  1 -
 source/blender/blenfont/intern/blf_font.c          | 96 +++++++++++++---------
 source/blender/blenfont/intern/blf_glyph.c         | 50 -----------
 source/blender/blenfont/intern/blf_internal.h      |  4 -
 .../blender/blenfont/intern/blf_internal_types.h   |  9 +-
 5 files changed, 62 insertions(+), 98 deletions(-)

diff --git a/source/blender/blenfont/intern/blf.c b/source/blender/blenfont/intern/blf.c
index 9168e7aa19c..2f9eb0753ac 100644
--- a/source/blender/blenfont/intern/blf.c
+++ b/source/blender/blenfont/intern/blf.c
@@ -108,7 +108,6 @@ void BLF_cache_clear(void)
     FontBLF *font = global_font[i];
     if (font) {
       blf_glyph_cache_clear(font);
-      blf_kerning_cache_clear(font);
     }
   }
 }
diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c
index 00d3cfb09eb..02847b74df1 100644
--- a/source/blender/blenfont/intern/blf_font.c
+++ b/source/blender/blenfont/intern/blf_font.c
@@ -308,17 +308,6 @@ void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi)
   blf_glyph_cache_release(font);
 }
 
-static void blf_font_ensure_ascii_kerning(FontBLF *font, GlyphCacheBLF *gc)
-{
-  if (font->kerning_cache || !FT_HAS_KERNING(font->face)) {
-    return;
-  }
-  font->kerning_cache = blf_kerning_cache_find(font);
-  if (!font->kerning_cache) {
-    font->kerning_cache = blf_kerning_cache_new(font, gc);
-  }
-}
-
 /* Fast path for runs of ASCII characters. Given that common UTF-8
  * input will consist of an overwhelming majority of ASCII
  * characters.
@@ -348,6 +337,31 @@ BLI_INLINE GlyphBLF *blf_utf8_next_fast(
   return g;
 }
 
+/* Convert a FreeType 26.6 value representing an unscaled design size to pixels.
+ * This is an exact copy of the scaling done inside FT_Get_Kerning when called
+ * with FT_KERNING_DEFAULT, including arbitrary resizing for small fonts.
+ */
+static int blf_unscaled_F26Dot6_to_pixels(FontBLF *font, FT_Pos value)
+{
+  /* Scale value by font size using integer-optimized multiplication. */
+  FT_Long scaled = FT_MulFix(value, font->face->size->metrics.x_scale);
+
+  /* FreeType states that this '25' has been determined heuristically. */
+  if (font->face->size->metrics.x_ppem < 25) {
+    scaled = FT_MulDiv(scaled, font->face->size->metrics.x_ppem, 25);
+  }
+
+  /* Copies of internal FreeType macros needed here. */
+#define FT_PIX_FLOOR(x) ((x) & ~FT_TYPEOF(x) 63)
+#define FT_PIX_ROUND(x) FT_PIX_FLOOR((x) + 32)
+
+  /* Round to even 64ths, then divide by 64. */
+  return FT_PIX_ROUND(scaled) >> 6;
+
+#undef FT_PIX_FLOOR
+#undef FT_PIX_ROUND
+}
+
 BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
                                       const GlyphBLF *g_prev,
                                       const GlyphBLF *g,
@@ -355,18 +369,28 @@ BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
                                       const uint c,
                                       int *pen_x_p)
 {
-  /* `blf_font_ensure_ascii_kerning(font, gc);` must be called before this function. */
-  BLI_assert(font->kerning_cache != NULL || !FT_HAS_KERNING(font->face));
+  if (FT_HAS_KERNING(font->face) && g_prev != NULL) {
+    FT_Vector delta = {KERNING_ENTRY_UNSET};
 
-  if (g_prev != NULL && FT_HAS_KERNING(font->face)) {
+    /* Get unscaled kerning value from our cache if ASCII. */
     if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
-      *pen_x_p += font->kerning_cache->ascii_table[c][c_prev];
+      delta.x = font->kerning_cache->ascii_table[c][c_prev];
     }
-    else {
-      FT_Vector delta;
-      if (FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_DEFAULT, &delta) == 0) {
-        *pen_x_p += (int)delta.x >> 6;
-      }
+
+    /* If not ASCII or not found in cache, ask FreeType for kerning. */
+    if (UNLIKELY(delta.x == KERNING_ENTRY_UNSET)) {
+      /* Note that this function sets delta values to zero on any error. */
+      FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_UNSCALED, &delta);
+    }
+
+    /* If ASCII we save this value to our cache for quicker access next time. */
+    if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
+      font->kerning_cache->ascii_table[c][c_prev] = delta.x;
+    }
+
+    if (delta.x != 0) {
+      /* Convert unscaled design units to pixels and move pen. */
+      *pen_x_p += blf_unscaled_F26Dot6_to_pixels(font, delta.x);
     }
   }
 }
@@ -388,8 +412,6 @@ static void blf_font_draw_ex(FontBLF *font,
     return;
   }
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   blf_batch_draw_begin(font);
 
   while ((i < len) && str[i]) {
@@ -435,8 +457,6 @@ static void blf_font_draw_ascii_ex(
 
   GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   blf_batch_draw_begin(font);
 
   while ((c = *(str++)) && len--) {
@@ -536,8 +556,6 @@ static void blf_font_draw_buffer_ex(FontBLF *font,
   int chx, chy;
   int y, x;
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   /* another buffer specific call for color conversion */
 
   while ((i < len) && str[i]) {
@@ -694,8 +712,6 @@ size_t blf_font_width_to_strlen(
   GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
   const int width_i = (int)width;
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   for (i_prev = i = 0, width_new = pen_x = 0, g_prev = NULL, c_prev = 0; (i < len) && str[i];
        i_prev = i, width_new = pen_x, c_prev = c, g_prev = g) {
     g = blf_utf8_next_fast(font, gc, str, &i, &c);
@@ -725,8 +741,6 @@ size_t blf_font_width_to_rstrlen(
   GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
   const int width_i = (int)width;
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   i = BLI_strnlen(str, len);
   s = BLI_str_find_prev_char_utf8(str, &str[i]);
   i = (size_t)((s != NULL) ? s - str : 0);
@@ -778,8 +792,6 @@ static void blf_font_boundbox_ex(FontBLF *font,
   box->ymin = 32000.0f;
   box->ymax = -32000.0f;
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   while ((i < len) && str[i]) {
     g = blf_utf8_next_fast(font, gc, str, &i, &c);
 
@@ -869,8 +881,6 @@ static void blf_font_wrap_apply(FontBLF *font,
 
   GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   struct WordWrapVars {
     int wrap_width;
     size_t start, last[2];
@@ -1123,8 +1133,6 @@ static void blf_font_boundbox_foreach_glyph_ex(FontBLF *font,
     return;
   }
 
-  blf_font_ensure_ascii_kerning(font, gc);
-
   while ((i < len) && str[i]) {
     i_curr = i;
     g = blf_utf8_next_fast(font, gc, str, &i, &c);
@@ -1205,7 +1213,9 @@ void blf_font_free(FontBLF *font)
     blf_glyph_cache_free(gc);
   }
 
-  blf_kerning_cache_clear(font);
+  if (font->kerning_cache) {
+    MEM_freeN(font->kerning_cache);
+  }
 
   FT_Done_Face(font->face);
   if (font->filename) {
@@ -1246,7 +1256,6 @@ static void blf_font_fill(FontBLF *font)
   font->dpi = 0;
   font->size = 0;
   BLI_listbase_clear(&font->cache);
-  BLI_listbase_clear(&font->kerning_caches);
   font->kerning_cache = NULL;
 #if BLF_BLUR_ENABLE
   font->blur = 0;
@@ -1307,6 +1316,17 @@ FontBLF *blf_font_new(const char *name, const char *filename)
   font->name = BLI_strdup(name);
   font->filename = BLI_strdup(filename);
   blf_font_fill(font);
+
+  if (FT_HAS_KERNING(font->face)) {
+    /* Create kerning cache table and fill with value indicating "unset". */
+    font->kerning_cache = MEM_mallocN(sizeof(KerningCacheBLF), __func__);
+    for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) {
+      for (uint j = 0; j < KERNING_CACHE_TABLE_SIZE; j++) {
+        font->kerning_cache->ascii_table[i][j] = KERNING_ENTRY_UNSET;
+      }
+    }
+  }
+
   return font;
 }
 
diff --git a/source/blender/blenfont/intern/blf_glyph.c b/source/blender/blenfont/intern/blf_glyph.c
index 5fb69251466..6cdf5fc5996 100644
--- a/source/blender/blenfont/intern/blf_glyph.c
+++ b/source/blender/blenfont/intern/blf_glyph.c
@@ -55,56 +55,6 @@
 #include "BLI_math_vector.h"
 #include "BLI_strict_flags.h"
 
-KerningCacheBLF *blf_kerning_cache_find(FontBLF *font)
-{
-  return (KerningCacheBLF *)font->kerning_caches.first;
-}
-
-/* Create a new glyph cache for the current kerning mode. */
-KerningCacheBLF *blf_kerning_cache_new(FontBLF *font, GlyphCacheBLF *gc)
-{
-  KerningCacheBLF *kc = MEM_mallocN(sizeof(KerningCacheBLF), __func__);
-  kc->next = NULL;
-  kc->prev = NULL;
-
-  GlyphBLF *g_table[KERNING_CACHE_TABLE_SIZE];
-  for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) {
-    GlyphBLF *g = blf_glyph_search(gc, i);
-    if (UNLIKELY(g == NULL)) {
-      FT_UInt glyph_index = FT_Get_Char_Index(font->face, i);
-      g = blf_glyph_add(font, gc, glyph_index, i);
-    }
-    g_table[i] = g;
-  }
-
-  memset(kc->ascii_table, 0, sizeof(kc->ascii_table));
-  for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) {
-    GlyphBLF *g = g_table[i];
-    if (g == NULL) {
-      continue;
-    }
-    for (uint j = 0; j < KERNING_CACHE_TABLE_SIZE; j++) {
-      GlyphBLF *g_prev = g_table[j];
-      if (g_prev == NULL) {
-        continue;
-      }
-      FT_Vector delta;
-      if (FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_DEFAULT, &delta) == 0) {
-        kc->ascii_table[i][j] = (int)delta.x >> 6;
-      }
-    }
-  }
-
-  BLI_addhead(&font->kerning_caches, kc);
-  return kc;
-}
-
-void blf_kerning_cache_clear(FontBLF *font)
-{
-  font->kerning_cache = NULL;
-  BLI_freelistN(&font->kerning_caches);
-}
-
 GlyphCacheBLF *blf_glyph_cache_find(FontBLF *font, unsigned int size, unsigned int dpi)
 {
   GlyphCacheBLF *p;
diff --git a/source/blender/blenfont/intern/blf_internal.h b/source/blender/blenfont/intern/blf_internal.h
index 35a6d019eac..ab2a26b1e06 100644
--- a/source/blender/blenfont/intern/blf_internal.h
+++ b/source/blender/blenfont/intern/blf_internal.h
@@ -121,10 +121,6 @@ int blf_font_count_missing_chars(struct FontBLF *font,
 
 void blf_font_free(struct FontBLF *font);
 
-struct KerningCacheBLF *blf_kerning_cache_find(struct FontBLF *font);
-struct KerningCacheBLF *blf_kerning_cache_new(struct FontBLF *font, struct GlyphCacheBLF *gc);
-void blf_kerning_cache_clear(struct FontBLF *font);
-
 struct GlyphCacheBLF *blf_glyph_cache_find(struct FontBLF *font,
                                            unsigned int size,
                                            unsigned int dpi);
diff --git a/source/blender/blenfont/intern/blf_internal_types.h b/source/blender/blenfont/intern/blf_internal_types.h
index caa10b2b125..38d7d7b6e21 100644
--- a/source/blender/blenfont/intern/blf_internal_types.h
+++ b/source/blender/blenfont/intern/blf_internal_types.h
@@ -34,6 +34,9 @@
 /* Number of characters in KerningCacheBLF.table. */
 #define KERNING_CACHE_TABLE_SIZE 128
 
+/* A value in the kerning cache that indicates it is not yet set. */
+#define KERNING_ENTRY_UNSET INT_MAX
+
 typedef struct BatchBLF {
   struct FontBLF *font; /* can only batch glyph from the same font */
   struct GPUBatch *batch;
@@ -50,7 +53,6 @@ typedef struct BatchBLF {
 extern BatchBLF g_batch;
 
 typedef struct KerningCacheBLF {
-  struct KerningCacheBLF *next, *prev;
   /**
    * Cache a ascii glyph pairs. Only store the x offset we are interested in,
    * instead of the full #FT_Vector since it's not used for drawing at the moment.
@@ -223,10 +225,7 @@ typedef struct FontBLF {
    */
   ListBase cache;
 
-  /* list of kerning cache for this font. */
-  ListBase kerning_caches;
-
-  /* current kerning cache for this font and kerning mode. */
+  /* Cache of unscaled kerning values. Will be NULL if font does not have kerning. */
   KerningCacheBLF *kerning_cache;
 
   /* freetype2 lib handle. */
-- 
cgit v1.2.3


From aed5a27755531aeef1e6f7a19dbde066cd9f2aba Mon Sep 17 00:00:00 2001
From: Campbell Barton <ideasman42@gmail.com>
Date: Sat, 21 Aug 2021 13:22:47 +1000
Subject: Correct build error from 0d7aab2375e6bb06e89dad851550b283a1ff805c

---
 source/blender/blenfont/intern/blf_font.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c
index 02847b74df1..436c4712e25 100644
--- a/source/blender/blenfont/intern/blf_font.c
+++ b/source/blender/blenfont/intern/blf_font.c
@@ -352,11 +352,11 @@ static int blf_unscaled_F26Dot6_to_pixels(FontBLF *font, FT_Pos value)
   }
 
   /* Copies of internal FreeType macros needed here. */
-#define FT_PIX_FLOOR(x) ((x) & ~FT_TYPEOF(x) 63)
+#define FT_PIX_FLOOR(x) ((x) & ~63)
 #define FT_PIX_ROUND(x) FT_PIX_FLOOR((x) + 32)
 
   /* Round to even 64ths, then divide by 64. */
-  return FT_PIX_ROUND(scaled) >> 6;
+  return (int)FT_PIX_ROUND(scaled) >> 6;
 
 #undef FT_PIX_FLOOR
 #undef FT_PIX_ROUND
@@ -385,7 +385,7 @@ BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
 
     /* If ASCII we save this value to our cache for quicker access next time. */
     if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
-      font->kerning_cache->ascii_table[c][c_prev] = delta.x;
+      font->kerning_cache->ascii_table[c][c_prev] = (int)delta.x;
     }
 
     if (delta.x != 0) {
-- 
cgit v1.2.3


From c671bfe14e8d8c72a146b74e4dcc1c8ee7ab5bb4 Mon Sep 17 00:00:00 2001
From: Campbell Barton <ideasman42@gmail.com>
Date: Sat, 21 Aug 2021 13:23:39 +1000
Subject: Cleanup: spelling in comments & minor cleanup

Also hyphenate 'mouse-move' use doxy sections in render_update.c &
move function comment from the header to the source.
---
 intern/ghost/GHOST_IEvent.h                        |  6 ++--
 source/blender/blenkernel/BKE_cachefile.h          |  5 ----
 source/blender/blenkernel/intern/cachefile.c       |  8 +++++-
 source/blender/draw/engines/eevee/eevee_lookdev.c  |  2 +-
 .../blender/editors/gpencil/gpencil_interpolate.c  |  4 +--
 source/blender/editors/gpencil/gpencil_primitive.c |  4 +--
 .../blender/editors/interface/interface_handlers.c |  8 +++---
 source/blender/editors/mesh/editmesh_tools.c       |  2 +-
 source/blender/editors/physics/particle_edit.c     |  2 +-
 source/blender/editors/render/render_update.c      | 32 ++++++++++++++--------
 .../editors/space_view3d/view3d_gizmo_ruler.c      |  4 ++-
 .../editors/transform/transform_convert_armature.c |  4 +--
 .../editors/transform/transform_convert_object.c   |  2 +-
 .../editors/transform/transform_snap_animation.c   |  4 +--
 source/blender/io/collada/collada_internal.cpp     |  2 +-
 .../blender/windowmanager/intern/wm_event_system.c |  4 +--
 16 files changed, 53 insertions(+), 40 deletions(-)

diff --git a/intern/ghost/GHOST_IEvent.h b/intern/ghost/GHOST_IEvent.h
index bcccd536ebd..239eea21088 100644
--- a/intern/ghost/GHOST_IEvent.h
+++ b/intern/ghost/GHOST_IEvent.h
@@ -32,10 +32,10 @@ class GHOST_IWindow;
 /**
  * Interface class for events received from GHOST.
  * You should not need to inherit this class. The system will pass these events
- * to the GHOST_IEventConsumer::processEvent() method of event consumers.<br>
- * Use the getType() method to retrieve the type of event and the getData()
+ * to the #GHOST_IEventConsumer::processEvent() method of event consumers.<br>
+ * Use the #getType() method to retrieve the type of event and the #getData()
  * method to get the event data out. Using the event type you can cast the
- * event data to the correct event dat structure.
+ * event data to the correct event data structure.
  * \see GHOST_IEventConsumer#processEvent
  * \see GHOST_TEventType
  */
diff --git a/source/blender/blenkernel/BKE_cachefile.h b/source/blender/blenkernel/BKE_cachefile.h
index 58d876b184b..836597f95da 100644
--- a/source/blender/blenkernel/BKE_cachefile.h
+++ b/source/blender/blenkernel/BKE_cachefile.h
@@ -61,11 +61,6 @@ void BKE_cachefile_reader_open(struct CacheFile *cache_file,
                                const char *object_path);
 void BKE_cachefile_reader_free(struct CacheFile *cache_file, struct CacheReader **reader);
 
-/* Determine whether the CacheFile should use a render engine procedural. If so, data is not read
- * from the file and bouding boxes are used to represent the objects in the Scene. Render engines
- * will receive the bounding box as a placeholder but can instead load the data directly if they
- * support it.
- */
 bool BKE_cache_file_uses_render_procedural(const struct CacheFile *cache_file,
                                            struct Scene *scene,
                                            const int dag_eval_mode);
diff --git a/source/blender/blenkernel/intern/cachefile.c b/source/blender/blenkernel/intern/cachefile.c
index 4a60564b4a1..87b1584d422 100644
--- a/source/blender/blenkernel/intern/cachefile.c
+++ b/source/blender/blenkernel/intern/cachefile.c
@@ -368,7 +368,7 @@ void BKE_cachefile_eval(Main *bmain, Depsgraph *depsgraph, CacheFile *cache_file
 #endif
 
   if (DEG_is_active(depsgraph)) {
-    /* Flush object paths back to original datablock for UI. */
+    /* Flush object paths back to original data-block for UI. */
     CacheFile *cache_file_orig = (CacheFile *)DEG_get_original_id(&cache_file->id);
     BLI_freelistN(&cache_file_orig->object_paths);
     BLI_duplicatelist(&cache_file_orig->object_paths, &cache_file->object_paths);
@@ -411,6 +411,12 @@ float BKE_cachefile_time_offset(const CacheFile *cache_file, const float time, c
   return cache_file->is_sequence ? frame : frame / fps - time_offset;
 }
 
+/**
+ * Determine whether the #CacheFile should use a render engine procedural. If so, data is not read
+ * from the file and bounding boxes are used to represent the objects in the Scene.
+ * Render engines will receive the bounding box as a placeholder but can instead
+ * load the data directly if they support it.
+ */
 bool BKE_cache_file_uses_render_procedural(const CacheFile *cache_file,
                                            Scene *scene,
                                            const int dag_eval_mode)
diff --git a/source/blender/draw/engines/eevee/eevee_lookdev.c b/source/blender/draw/engines/eevee/eevee_lookdev.c
index f4dc553e982..879a7b08eba 100644
--- a/source/blender/draw/engines/eevee/eevee_lookdev.c
+++ b/source/blender/draw/engines/eevee/eevee_lookdev.c
@@ -246,7 +246,7 @@ void EEVEE_lookdev_cache_init(EEVEE_Data *vedata,
       DRW_shgroup_uniform_float_copy(grp, "studioLightIntensity", shading->studiolight_intensity);
       BKE_studiolight_ensure_flag(sl, STUDIOLIGHT_EQUIRECT_RADIANCE_GPUTEXTURE);
       DRW_shgroup_uniform_texture_ex(grp, "studioLight", sl->equirect_radiance_gputexture, state);
-      /* Do not fadeout when doing probe rendering, only when drawing the background */
+      /* Do not fade-out when doing probe rendering, only when drawing the background. */
       DRW_shgroup_uniform_float_copy(grp, "backgroundAlpha", 1.0f);
     }
     else {
diff --git a/source/blender/editors/gpencil/gpencil_interpolate.c b/source/blender/editors/gpencil/gpencil_interpolate.c
index 8640ffa67cf..a8bd3b11bb1 100644
--- a/source/blender/editors/gpencil/gpencil_interpolate.c
+++ b/source/blender/editors/gpencil/gpencil_interpolate.c
@@ -890,9 +890,9 @@ static int gpencil_interpolate_modal(bContext *C, wmOperator *op, const wmEvent
     }
     case MOUSEMOVE: /* calculate new position */
     {
-      /* only handle mousemove if not doing numinput */
+      /* Only handle mouse-move if not doing numeric-input. */
       if (has_numinput == false) {
-        /* update shift based on position of mouse */
+        /* Update shift based on position of mouse. */
         gpencil_mouse_update_shift(tgpi, op, event);
 
         /* update screen */
diff --git a/source/blender/editors/gpencil/gpencil_primitive.c b/source/blender/editors/gpencil/gpencil_primitive.c
index 894fb83d70e..5ecb6d9a212 100644
--- a/source/blender/editors/gpencil/gpencil_primitive.c
+++ b/source/blender/editors/gpencil/gpencil_primitive.c
@@ -1944,9 +1944,9 @@ static int gpencil_primitive_modal(bContext *C, wmOperator *op, const wmEvent *e
       if (ELEM(tgpi->flag, IN_CURVE_EDIT)) {
         break;
       }
-      /* only handle mousemove if not doing numinput */
+      /* Only handle mouse-move if not doing numeric-input. */
       if (has_numinput == false) {
-        /* update position of mouse */
+        /* Update position of mouse. */
         copy_v2_v2(tgpi->end, tgpi->mval);
         copy_v2_v2(tgpi->start, tgpi->origin);
         if (tgpi->flag == IDLE) {
diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c
index b8f324cba83..76f6640c714 100644
--- a/source/blender/editors/interface/interface_handlers.c
+++ b/source/blender/editors/interface/interface_handlers.c
@@ -6025,7 +6025,7 @@ static int ui_do_but_BLOCK(bContext *C, uiBut *but, uiHandleButtonData *data, co
          * the slot menu fails to switch a second time.
          *
          * The active state of the button could be maintained some other way
-         * and remove this mousemove event.
+         * and remove this mouse-move event.
          */
         WM_event_add_mousemove(data->window);
 
@@ -8364,7 +8364,7 @@ static void button_activate_state(bContext *C, uiBut *but, uiHandleButtonState s
     }
   }
 
-  /* wait for mousemove to enable drag */
+  /* Wait for mouse-move to enable drag. */
   if (state == BUTTON_STATE_WAIT_DRAG) {
     but->flag &= ~UI_SELECT;
   }
@@ -8631,9 +8631,9 @@ static void button_activate_exit(
     ui_but_update(but);
   }
 
-  /* adds empty mousemove in queue for re-init handler, in case mouse is
+  /* Adds empty mouse-move in queue for re-initialize handler, in case mouse is
    * still over a button. We cannot just check for this ourselves because
-   * at this point the mouse may be over a button in another region */
+   * at this point the mouse may be over a button in another region. */
   if (mousemove) {
     WM_event_add_mousemove(CTX_wm_window(C));
   }
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index b8bbf2d3e70..956658bd2b7 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -8627,7 +8627,7 @@ static int edbm_point_normals_modal(bContext *C, wmOperator *op, const wmEvent *
     RNA_enum_set(op->ptr, "mode", mode);
   }
 
-  /* Only handle mousemove event in case we are in mouse mode. */
+  /* Only handle mouse-move event in case we are in mouse mode. */
   if (event->type == MOUSEMOVE || force_mousemove) {
     if (mode == EDBM_CLNOR_POINTTO_MODE_MOUSE) {
       ARegion *region = CTX_wm_region(C);
diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index 85883a2d29a..8afc5c583e0 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -4667,7 +4667,7 @@ typedef struct BrushEdit {
   int lastmouse[2];
   float zfac;
 
-  /* optional cached view settings to avoid setting on every mousemove */
+  /** Optional cached view settings to avoid setting on every mouse-move. */
   PEData data;
 } BrushEdit;
 
diff --git a/source/blender/editors/render/render_update.c b/source/blender/editors/render/render_update.c
index 6db148eb4e1..8bc2281db73 100644
--- a/source/blender/editors/render/render_update.c
+++ b/source/blender/editors/render/render_update.c
@@ -64,7 +64,9 @@
 
 #include <stdio.h>
 
-/***************************** Render Engines ********************************/
+/* -------------------------------------------------------------------- */
+/** \name Render Engines
+ * \{ */
 
 /* Update 3D viewport render or draw engine on changes to the scene or view settings. */
 void ED_render_view3d_update(Depsgraph *depsgraph,
@@ -206,15 +208,15 @@ void ED_render_engine_changed(Main *bmain, const bool update_scene_data)
     }
   }
 
-  /* Update CacheFiles to ensure that procedurals are properly taken into account. */
+  /* Update #CacheFiles to ensure that procedurals are properly taken into account. */
   LISTBASE_FOREACH (CacheFile *, cachefile, &bmain->cachefiles) {
-    /* Only update cachefiles which are set to use a render procedural. We do not use
-     * BKE_cachefile_uses_render_procedural here as we need to update regardless of the current
-     * engine or its settings. */
+    /* Only update cache-files which are set to use a render procedural.
+     * We do not use #BKE_cachefile_uses_render_procedural here as we need to update regardless of
+     * the current engine or its settings. */
     if (cachefile->use_render_procedural) {
       DEG_id_tag_update(&cachefile->id, ID_RECALC_COPY_ON_WRITE);
-      /* Rebuild relations so that modifiers are reconnected to or disconnected from the cachefile.
-       */
+      /* Rebuild relations so that modifiers are reconnected to or disconnected from the
+       * cache-file. */
       DEG_relations_tag_update(bmain);
     }
   }
@@ -227,10 +229,16 @@ void ED_render_view_layer_changed(Main *bmain, bScreen *screen)
   }
 }
 
-/***************************** Updates ***********************************
- * ED_render_id_flush_update gets called from DEG_id_tag_update, to do   *
- * editor level updates when the ID changes. when these ID blocks are in *
- * the dependency graph, we can get rid of the manual dependency checks. */
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Updates
+ *
+ * #ED_render_id_flush_update gets called from #DEG_id_tag_update,
+ * to do editor level updates when the ID changes.
+ * When these ID blocks are in the dependency graph,
+ * we can get rid of the manual dependency checks.
+ * \{ */
 
 static void material_changed(Main *UNUSED(bmain), Material *ma)
 {
@@ -336,3 +344,5 @@ void ED_render_id_flush_update(const DEGEditorUpdateContext *update_ctx, ID *id)
       break;
   }
 }
+
+/** \} */
diff --git a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
index 49299d73337..edc34d0d883 100644
--- a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
+++ b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
@@ -299,7 +299,9 @@ static void view3d_ruler_item_project(RulerInfo *ruler_info, float r_co[3], cons
   ED_view3d_win_to_3d_int(ruler_info->area->spacedata.first, ruler_info->region, r_co, xy, r_co);
 }
 
-/* use for mousemove events */
+/**
+ * Use for mouse-move events.
+ */
 static bool view3d_ruler_item_mousemove(struct Depsgraph *depsgraph,
                                         RulerInfo *ruler_info,
                                         RulerItem *ruler_item,
diff --git a/source/blender/editors/transform/transform_convert_armature.c b/source/blender/editors/transform/transform_convert_armature.c
index f56d60b7376..5627a910ab4 100644
--- a/source/blender/editors/transform/transform_convert_armature.c
+++ b/source/blender/editors/transform/transform_convert_armature.c
@@ -131,12 +131,12 @@ static void autokeyframe_pose(
 
     ListBase dsources = {NULL, NULL};
 
-    /* add datasource override for the camera object */
+    /* Add data-source override for the camera object. */
     ANIM_relative_keyingset_add_source(&dsources, id, &RNA_PoseBone, pchan);
 
     /* only insert into active keyingset? */
     if (IS_AUTOKEY_FLAG(scene, ONLYKEYINGSET) && (active_ks)) {
-      /* run the active Keying Set on the current datasource */
+      /* Run the active Keying Set on the current data-source. */
       ANIM_apply_keyingset(
           C, &dsources, NULL, active_ks, MODIFYKEY_MODE_INSERT, anim_eval_context.eval_time);
     }
diff --git a/source/blender/editors/transform/transform_convert_object.c b/source/blender/editors/transform/transform_convert_object.c
index ee6cb391fdc..bcbac009948 100644
--- a/source/blender/editors/transform/transform_convert_object.c
+++ b/source/blender/editors/transform/transform_convert_object.c
@@ -749,7 +749,7 @@ static void autokeyframe_object(
     /* Get flags used for inserting keyframes. */
     flag = ANIM_get_keyframing_flags(scene, true);
 
-    /* add datasource override for the object */
+    /* Add data-source override for the object. */
     ANIM_relative_keyingset_add_source(&dsources, id, NULL, NULL);
 
     if (IS_AUTOKEY_FLAG(scene, ONLYKEYINGSET) && (active_ks)) {
diff --git a/source/blender/editors/transform/transform_snap_animation.c b/source/blender/editors/transform/transform_snap_animation.c
index be7fc65a6c2..08335924ddf 100644
--- a/source/blender/editors/transform/transform_snap_animation.c
+++ b/source/blender/editors/transform/transform_snap_animation.c
@@ -35,14 +35,14 @@
 #include "transform_snap.h"
 
 /* -------------------------------------------------------------------- */
-/** \name Snappint in Anim Editors
+/** \name Snapping in Anim Editors
  * \{ */
 
 /**
  * This function returns the snapping 'mode' for Animation Editors only.
  * We cannot use the standard snapping due to NLA-strip scaling complexities.
  *
- * TODO: these modifier checks should be key-mappable.
+ * TODO: these modifier checks should be accessible from the key-map.
  */
 short getAnimEdit_SnapMode(TransInfo *t)
 {
diff --git a/source/blender/io/collada/collada_internal.cpp b/source/blender/io/collada/collada_internal.cpp
index 355aa5c22f0..bd6f496c8ec 100644
--- a/source/blender/io/collada/collada_internal.cpp
+++ b/source/blender/io/collada/collada_internal.cpp
@@ -162,7 +162,7 @@ void UnitConverter::calculate_scale(Scene &sce)
  * Translation map.
  * Used to translate every COLLADA id to a valid id, no matter what "wrong" letters may be
  * included. Look at the IDREF XSD declaration for more.
- * Follows strictly the COLLADA XSD declaration which explicitly allows non-english chars,
+ * Follows strictly the COLLADA XSD declaration which explicitly allows non-English chars,
  * like special chars (e.g. micro sign), umlauts and so on.
  * The COLLADA spec also allows additional chars for member access ('.'), these
  * must obviously be removed too, otherwise they would be heavily misinterpreted.
diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c
index 5cc361fc1bd..b9a3dd0c3fb 100644
--- a/source/blender/windowmanager/intern/wm_event_system.c
+++ b/source/blender/windowmanager/intern/wm_event_system.c
@@ -3471,8 +3471,8 @@ void wm_event_do_handlers(bContext *C)
             }
             CTX_wm_area_set(C, NULL);
 
-            /* NOTE: do not escape on WM_HANDLER_BREAK,
-             * mousemove needs handled for previous area. */
+            /* NOTE: do not escape on #WM_HANDLER_BREAK,
+             * mouse-move needs handled for previous area. */
           }
         }
 
-- 
cgit v1.2.3


From 47e68537f83075dd78ed66aac3216643389b0e3a Mon Sep 17 00:00:00 2001
From: Campbell Barton <ideasman42@gmail.com>
Date: Sat, 21 Aug 2021 17:41:40 +1000
Subject: Cleanup: organize blf_font.c functions using doxy-sections

Functions in this file were scattered and not well organized.
---
 source/blender/blenfont/intern/blf_font.c | 743 ++++++++++++++++--------------
 1 file changed, 403 insertions(+), 340 deletions(-)

diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c
index 436c4712e25..47fa3544474 100644
--- a/source/blender/blenfont/intern/blf_font.c
+++ b/source/blender/blenfont/intern/blf_font.c
@@ -71,6 +71,38 @@ static FT_Library ft_lib;
 static SpinLock ft_lib_mutex;
 static SpinLock blf_glyph_cache_mutex;
 
+/* -------------------------------------------------------------------- */
+/** \name FreeType Utilities (Internal)
+ * \{ */
+
+/**
+ * Convert a FreeType 26.6 value representing an unscaled design size to pixels.
+ * This is an exact copy of the scaling done inside FT_Get_Kerning when called
+ * with #FT_KERNING_DEFAULT, including arbitrary resizing for small fonts.
+ */
+static int blf_unscaled_F26Dot6_to_pixels(FontBLF *font, FT_Pos value)
+{
+  /* Scale value by font size using integer-optimized multiplication. */
+  FT_Long scaled = FT_MulFix(value, font->face->size->metrics.x_scale);
+
+  /* FreeType states that this '25' has been determined heuristically. */
+  if (font->face->size->metrics.x_ppem < 25) {
+    scaled = FT_MulDiv(scaled, font->face->size->metrics.x_ppem, 25);
+  }
+
+  /* Copies of internal FreeType macros needed here. */
+#define FT_PIX_FLOOR(x) ((x) & ~63)
+#define FT_PIX_ROUND(x) FT_PIX_FLOOR((x) + 32)
+
+  /* Round to even 64ths, then divide by 64. */
+  return (int)FT_PIX_ROUND(scaled) >> 6;
+
+#undef FT_PIX_FLOOR
+#undef FT_PIX_ROUND
+}
+
+/** \} */
+
 /* -------------------------------------------------------------------- */
 /** \name Glyph Batching
  * \{ */
@@ -257,56 +289,8 @@ static void blf_batch_draw_end(void)
 /** \} */
 
 /* -------------------------------------------------------------------- */
-
-int blf_font_init(void)
-{
-  memset(&g_batch, 0, sizeof(g_batch));
-  BLI_spin_init(&ft_lib_mutex);
-  BLI_spin_init(&blf_glyph_cache_mutex);
-  return FT_Init_FreeType(&ft_lib);
-}
-
-void blf_font_exit(void)
-{
-  FT_Done_FreeType(ft_lib);
-  BLI_spin_end(&ft_lib_mutex);
-  BLI_spin_end(&blf_glyph_cache_mutex);
-  blf_batch_draw_exit();
-}
-
-void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi)
-{
-  GlyphCacheBLF *gc;
-  FT_Error err;
-
-  blf_glyph_cache_acquire(font);
-
-  gc = blf_glyph_cache_find(font, size, dpi);
-  if (gc) {
-    /* Optimization: do not call FT_Set_Char_Size if size did not change. */
-    if (font->size == size && font->dpi == dpi) {
-      blf_glyph_cache_release(font);
-      return;
-    }
-  }
-
-  err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi);
-  if (err) {
-    /* FIXME: here we can go through the fixed size and choice a close one */
-    printf("The current font don't support the size, %u and dpi, %u\n", size, dpi);
-
-    blf_glyph_cache_release(font);
-    return;
-  }
-
-  font->size = size;
-  font->dpi = dpi;
-
-  if (!gc) {
-    blf_glyph_cache_new(font);
-  }
-  blf_glyph_cache_release(font);
-}
+/** \name Glyph Stepping Utilities (Internal)
+ * \{ */
 
 /* Fast path for runs of ASCII characters. Given that common UTF-8
  * input will consist of an overwhelming majority of ASCII
@@ -337,31 +321,6 @@ BLI_INLINE GlyphBLF *blf_utf8_next_fast(
   return g;
 }
 
-/* Convert a FreeType 26.6 value representing an unscaled design size to pixels.
- * This is an exact copy of the scaling done inside FT_Get_Kerning when called
- * with FT_KERNING_DEFAULT, including arbitrary resizing for small fonts.
- */
-static int blf_unscaled_F26Dot6_to_pixels(FontBLF *font, FT_Pos value)
-{
-  /* Scale value by font size using integer-optimized multiplication. */
-  FT_Long scaled = FT_MulFix(value, font->face->size->metrics.x_scale);
-
-  /* FreeType states that this '25' has been determined heuristically. */
-  if (font->face->size->metrics.x_ppem < 25) {
-    scaled = FT_MulDiv(scaled, font->face->size->metrics.x_ppem, 25);
-  }
-
-  /* Copies of internal FreeType macros needed here. */
-#define FT_PIX_FLOOR(x) ((x) & ~63)
-#define FT_PIX_ROUND(x) FT_PIX_FLOOR((x) + 32)
-
-  /* Round to even 64ths, then divide by 64. */
-  return (int)FT_PIX_ROUND(scaled) >> 6;
-
-#undef FT_PIX_FLOOR
-#undef FT_PIX_ROUND
-}
-
 BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
                                       const GlyphBLF *g_prev,
                                       const GlyphBLF *g,
@@ -395,6 +354,12 @@ BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
   }
 }
 
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Text Drawing: GPU
+ * \{ */
+
 static void blf_font_draw_ex(FontBLF *font,
                              GlyphCacheBLF *gc,
                              const char *str,
@@ -535,6 +500,12 @@ int blf_font_draw_mono(FontBLF *font, const char *str, size_t len, int cwidth)
   return columns;
 }
 
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Text Drawgin: Buffer
+ * \{ */
+
 /* Sanity checks are done by BLF_draw_buffer() */
 static void blf_font_draw_buffer_ex(FontBLF *font,
                                     GlyphCacheBLF *gc,
@@ -680,6 +651,16 @@ void blf_font_draw_buffer(FontBLF *font, const char *str, size_t len, struct Res
   blf_glyph_cache_release(font);
 }
 
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Text Evaluation: Width to Sting Length
+ *
+ * Use to implement exported functions:
+ * - #BLF_width_to_strlen
+ * - #BLF_width_to_rstrlen
+ * \{ */
+
 static bool blf_font_width_to_strlen_glyph_process(FontBLF *font,
                                                    const uint c_prev,
                                                    const uint c,
@@ -773,6 +754,12 @@ size_t blf_font_width_to_rstrlen(
   return i;
 }
 
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Text Evaluation: Glyph Bound Box with Callback
+ * \{ */
+
 static void blf_font_boundbox_ex(FontBLF *font,
                                  GlyphCacheBLF *gc,
                                  const char *str,
@@ -847,73 +834,230 @@ void blf_font_boundbox(
   blf_glyph_cache_release(font);
 }
 
-/* -------------------------------------------------------------------- */
-/** \name Word-Wrap Support
- * \{ */
+void blf_font_width_and_height(FontBLF *font,
+                               const char *str,
+                               size_t len,
+                               float *r_width,
+                               float *r_height,
+                               struct ResultBLF *r_info)
+{
+  float xa, ya;
+  rctf box;
 
-/**
- * Generic function to add word-wrap support for other existing functions.
- *
- * Wraps on spaces and respects newlines.
- * Intentionally ignores non-unix newlines, tabs and more advanced text formatting.
- *
- * \note If we want rich text - we better have a higher level API to handle that
- * (color, bold, switching fonts... etc).
- */
-static void blf_font_wrap_apply(FontBLF *font,
-                                const char *str,
-                                size_t len,
-                                struct ResultBLF *r_info,
-                                void (*callback)(FontBLF *font,
-                                                 GlyphCacheBLF *gc,
-                                                 const char *str,
-                                                 size_t len,
-                                                 int pen_y,
-                                                 void *userdata),
-                                void *userdata)
+  if (font->flags & BLF_ASPECT) {
+    xa = font->aspect[0];
+    ya = font->aspect[1];
+  }
+  else {
+    xa = 1.0f;
+    ya = 1.0f;
+  }
+
+  if (font->flags & BLF_WORD_WRAP) {
+    blf_font_boundbox__wrap(font, str, len, &box, r_info);
+  }
+  else {
+    blf_font_boundbox(font, str, len, &box, r_info);
+  }
+  *r_width = (BLI_rctf_size_x(&box) * xa);
+  *r_height = (BLI_rctf_size_y(&box) * ya);
+}
+
+float blf_font_width(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info)
 {
-  unsigned int c, c_prev = BLI_UTF8_ERR;
-  GlyphBLF *g, *g_prev = NULL;
-  int pen_x = 0, pen_y = 0;
-  size_t i = 0;
-  int lines = 0;
-  int pen_x_next = 0;
+  float xa;
+  rctf box;
 
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  if (font->flags & BLF_ASPECT) {
+    xa = font->aspect[0];
+  }
+  else {
+    xa = 1.0f;
+  }
 
-  struct WordWrapVars {
-    int wrap_width;
-    size_t start, last[2];
-  } wrap = {font->wrap_width != -1 ? font->wrap_width : INT_MAX, 0, {0, 0}};
+  if (font->flags & BLF_WORD_WRAP) {
+    blf_font_boundbox__wrap(font, str, len, &box, r_info);
+  }
+  else {
+    blf_font_boundbox(font, str, len, &box, r_info);
+  }
+  return BLI_rctf_size_x(&box) * xa;
+}
 
-  // printf("%s wrapping (%d, %d) `%s`:\n", __func__, len, strlen(str), str);
-  while ((i < len) && str[i]) {
+float blf_font_height(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info)
+{
+  float ya;
+  rctf box;
 
-    /* wrap vars */
-    size_t i_curr = i;
-    bool do_draw = false;
+  if (font->flags & BLF_ASPECT) {
+    ya = font->aspect[1];
+  }
+  else {
+    ya = 1.0f;
+  }
 
-    g = blf_utf8_next_fast(font, gc, str, &i, &c);
+  if (font->flags & BLF_WORD_WRAP) {
+    blf_font_boundbox__wrap(font, str, len, &box, r_info);
+  }
+  else {
+    blf_font_boundbox(font, str, len, &box, r_info);
+  }
+  return BLI_rctf_size_y(&box) * ya;
+}
 
-    if (UNLIKELY(c == BLI_UTF8_ERR)) {
-      break;
-    }
-    if (UNLIKELY(g == NULL)) {
-      continue;
-    }
-    blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x);
+float blf_font_fixed_width(FontBLF *font)
+{
+  const unsigned int c = ' ';
 
-    /**
-     * Implementation Detail (utf8).
-     *
-     * Take care with single byte offsets here,
-     * since this is utf8 we can't be sure a single byte is a single character.
-     *
-     * This is _only_ done when we know for sure the character is ascii (newline or a space).
-     */
-    pen_x_next = pen_x + g->advance_i;
-    if (UNLIKELY((pen_x_next >= wrap.wrap_width) && (wrap.start != wrap.last[0]))) {
-      do_draw = true;
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  GlyphBLF *g = blf_glyph_search(gc, c);
+  if (!g) {
+    g = blf_glyph_add(font, gc, FT_Get_Char_Index(font->face, c), c);
+
+    /* if we don't find the glyph. */
+    if (!g) {
+      blf_glyph_cache_release(font);
+      return 0.0f;
+    }
+  }
+
+  blf_glyph_cache_release(font);
+  return g->advance;
+}
+
+static void blf_font_boundbox_foreach_glyph_ex(FontBLF *font,
+                                               GlyphCacheBLF *gc,
+                                               const char *str,
+                                               size_t len,
+                                               BLF_GlyphBoundsFn user_fn,
+                                               void *user_data,
+                                               struct ResultBLF *r_info,
+                                               int pen_y)
+{
+  unsigned int c, c_prev = BLI_UTF8_ERR;
+  GlyphBLF *g, *g_prev = NULL;
+  int pen_x = 0;
+  size_t i = 0, i_curr;
+  rcti gbox;
+
+  if (len == 0) {
+    /* early output. */
+    return;
+  }
+
+  while ((i < len) && str[i]) {
+    i_curr = i;
+    g = blf_utf8_next_fast(font, gc, str, &i, &c);
+
+    if (UNLIKELY(c == BLI_UTF8_ERR)) {
+      break;
+    }
+    if (UNLIKELY(g == NULL)) {
+      continue;
+    }
+    blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x);
+
+    gbox.xmin = pen_x;
+    gbox.xmax = gbox.xmin + MIN2(g->advance_i, g->dims[0]);
+    gbox.ymin = pen_y;
+    gbox.ymax = gbox.ymin - g->dims[1];
+
+    pen_x += g->advance_i;
+
+    if (user_fn(str, i_curr, &gbox, g->advance_i, &g->box, g->pos, user_data) == false) {
+      break;
+    }
+
+    g_prev = g;
+    c_prev = c;
+  }
+
+  if (r_info) {
+    r_info->lines = 1;
+    r_info->width = pen_x;
+  }
+}
+void blf_font_boundbox_foreach_glyph(FontBLF *font,
+                                     const char *str,
+                                     size_t len,
+                                     BLF_GlyphBoundsFn user_fn,
+                                     void *user_data,
+                                     struct ResultBLF *r_info)
+{
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  blf_font_boundbox_foreach_glyph_ex(font, gc, str, len, user_fn, user_data, r_info, 0);
+  blf_glyph_cache_release(font);
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Text Evaluation: Word-Wrap with Callback
+ * \{ */
+
+/**
+ * Generic function to add word-wrap support for other existing functions.
+ *
+ * Wraps on spaces and respects newlines.
+ * Intentionally ignores non-unix newlines, tabs and more advanced text formatting.
+ *
+ * \note If we want rich text - we better have a higher level API to handle that
+ * (color, bold, switching fonts... etc).
+ */
+static void blf_font_wrap_apply(FontBLF *font,
+                                const char *str,
+                                size_t len,
+                                struct ResultBLF *r_info,
+                                void (*callback)(FontBLF *font,
+                                                 GlyphCacheBLF *gc,
+                                                 const char *str,
+                                                 size_t len,
+                                                 int pen_y,
+                                                 void *userdata),
+                                void *userdata)
+{
+  unsigned int c, c_prev = BLI_UTF8_ERR;
+  GlyphBLF *g, *g_prev = NULL;
+  int pen_x = 0, pen_y = 0;
+  size_t i = 0;
+  int lines = 0;
+  int pen_x_next = 0;
+
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+
+  struct WordWrapVars {
+    int wrap_width;
+    size_t start, last[2];
+  } wrap = {font->wrap_width != -1 ? font->wrap_width : INT_MAX, 0, {0, 0}};
+
+  // printf("%s wrapping (%d, %d) `%s`:\n", __func__, len, strlen(str), str);
+  while ((i < len) && str[i]) {
+
+    /* wrap vars */
+    size_t i_curr = i;
+    bool do_draw = false;
+
+    g = blf_utf8_next_fast(font, gc, str, &i, &c);
+
+    if (UNLIKELY(c == BLI_UTF8_ERR)) {
+      break;
+    }
+    if (UNLIKELY(g == NULL)) {
+      continue;
+    }
+    blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x);
+
+    /**
+     * Implementation Detail (utf8).
+     *
+     * Take care with single byte offsets here,
+     * since this is utf8 we can't be sure a single byte is a single character.
+     *
+     * This is _only_ done when we know for sure the character is ascii (newline or a space).
+     */
+    pen_x_next = pen_x + g->advance_i;
+    if (UNLIKELY((pen_x_next >= wrap.wrap_width) && (wrap.start != wrap.last[0]))) {
+      do_draw = true;
     }
     else if (UNLIKELY(((i < len) && str[i]) == 0)) {
       /* need check here for trailing newline, else we draw it */
@@ -1018,216 +1162,120 @@ void blf_font_draw_buffer__wrap(FontBLF *font,
 
 /** \} */
 
-void blf_font_width_and_height(FontBLF *font,
-                               const char *str,
-                               size_t len,
-                               float *r_width,
-                               float *r_height,
-                               struct ResultBLF *r_info)
+/* -------------------------------------------------------------------- */
+/** \name Text Evaluation: Count Missing Characters
+ * \{ */
+
+int blf_font_count_missing_chars(FontBLF *font,
+                                 const char *str,
+                                 const size_t len,
+                                 int *r_tot_chars)
 {
-  float xa, ya;
-  rctf box;
+  int missing = 0;
+  size_t i = 0;
 
-  if (font->flags & BLF_ASPECT) {
-    xa = font->aspect[0];
-    ya = font->aspect[1];
-  }
-  else {
-    xa = 1.0f;
-    ya = 1.0f;
-  }
+  *r_tot_chars = 0;
+  while (i < len) {
+    unsigned int c;
 
-  if (font->flags & BLF_WORD_WRAP) {
-    blf_font_boundbox__wrap(font, str, len, &box, r_info);
-  }
-  else {
-    blf_font_boundbox(font, str, len, &box, r_info);
+    if ((c = str[i]) < GLYPH_ASCII_TABLE_SIZE) {
+      i++;
+    }
+    else if ((c = BLI_str_utf8_as_unicode_step(str, &i)) != BLI_UTF8_ERR) {
+      if (FT_Get_Char_Index((font)->face, c) == 0) {
+        missing++;
+      }
+    }
+    (*r_tot_chars)++;
   }
-  *r_width = (BLI_rctf_size_x(&box) * xa);
-  *r_height = (BLI_rctf_size_y(&box) * ya);
+  return missing;
 }
 
-float blf_font_width(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info)
-{
-  float xa;
-  rctf box;
-
-  if (font->flags & BLF_ASPECT) {
-    xa = font->aspect[0];
-  }
-  else {
-    xa = 1.0f;
-  }
+/** \} */
 
-  if (font->flags & BLF_WORD_WRAP) {
-    blf_font_boundbox__wrap(font, str, len, &box, r_info);
-  }
-  else {
-    blf_font_boundbox(font, str, len, &box, r_info);
-  }
-  return BLI_rctf_size_x(&box) * xa;
-}
+/* -------------------------------------------------------------------- */
+/** \name Font Query: Attributes
+ * \{ */
 
-float blf_font_height(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info)
+int blf_font_height_max(FontBLF *font)
 {
-  float ya;
-  rctf box;
+  int height_max;
 
-  if (font->flags & BLF_ASPECT) {
-    ya = font->aspect[1];
-  }
-  else {
-    ya = 1.0f;
-  }
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  height_max = gc->glyph_height_max;
 
-  if (font->flags & BLF_WORD_WRAP) {
-    blf_font_boundbox__wrap(font, str, len, &box, r_info);
-  }
-  else {
-    blf_font_boundbox(font, str, len, &box, r_info);
-  }
-  return BLI_rctf_size_y(&box) * ya;
+  blf_glyph_cache_release(font);
+  return height_max;
 }
 
-float blf_font_fixed_width(FontBLF *font)
+int blf_font_width_max(FontBLF *font)
 {
-  const unsigned int c = ' ';
+  int width_max;
 
   GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  GlyphBLF *g = blf_glyph_search(gc, c);
-  if (!g) {
-    g = blf_glyph_add(font, gc, FT_Get_Char_Index(font->face, c), c);
-
-    /* if we don't find the glyph. */
-    if (!g) {
-      blf_glyph_cache_release(font);
-      return 0.0f;
-    }
-  }
+  width_max = gc->glyph_width_max;
 
   blf_glyph_cache_release(font);
-  return g->advance;
+  return width_max;
 }
 
-/* -------------------------------------------------------------------- */
-/** \name Glyph Bound Box with Callback
- * \{ */
-
-static void blf_font_boundbox_foreach_glyph_ex(FontBLF *font,
-                                               GlyphCacheBLF *gc,
-                                               const char *str,
-                                               size_t len,
-                                               BLF_GlyphBoundsFn user_fn,
-                                               void *user_data,
-                                               struct ResultBLF *r_info,
-                                               int pen_y)
+float blf_font_descender(FontBLF *font)
 {
-  unsigned int c, c_prev = BLI_UTF8_ERR;
-  GlyphBLF *g, *g_prev = NULL;
-  int pen_x = 0;
-  size_t i = 0, i_curr;
-  rcti gbox;
-
-  if (len == 0) {
-    /* early output. */
-    return;
-  }
-
-  while ((i < len) && str[i]) {
-    i_curr = i;
-    g = blf_utf8_next_fast(font, gc, str, &i, &c);
-
-    if (UNLIKELY(c == BLI_UTF8_ERR)) {
-      break;
-    }
-    if (UNLIKELY(g == NULL)) {
-      continue;
-    }
-    blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x);
+  float descender;
 
-    gbox.xmin = pen_x;
-    gbox.xmax = gbox.xmin + MIN2(g->advance_i, g->dims[0]);
-    gbox.ymin = pen_y;
-    gbox.ymax = gbox.ymin - g->dims[1];
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  descender = gc->descender;
 
-    pen_x += g->advance_i;
+  blf_glyph_cache_release(font);
+  return descender;
+}
 
-    if (user_fn(str, i_curr, &gbox, g->advance_i, &g->box, g->pos, user_data) == false) {
-      break;
-    }
+float blf_font_ascender(FontBLF *font)
+{
+  float ascender;
 
-    g_prev = g;
-    c_prev = c;
-  }
+  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
+  ascender = gc->ascender;
 
-  if (r_info) {
-    r_info->lines = 1;
-    r_info->width = pen_x;
-  }
+  blf_glyph_cache_release(font);
+  return ascender;
 }
-void blf_font_boundbox_foreach_glyph(FontBLF *font,
-                                     const char *str,
-                                     size_t len,
-                                     BLF_GlyphBoundsFn user_fn,
-                                     void *user_data,
-                                     struct ResultBLF *r_info)
+
+char *blf_display_name(FontBLF *font)
 {
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  blf_font_boundbox_foreach_glyph_ex(font, gc, str, len, user_fn, user_data, r_info, 0);
-  blf_glyph_cache_release(font);
+  if (!font->face->family_name) {
+    return NULL;
+  }
+  return BLI_sprintfN("%s %s", font->face->family_name, font->face->style_name);
 }
 
 /** \} */
 
-int blf_font_count_missing_chars(FontBLF *font,
-                                 const char *str,
-                                 const size_t len,
-                                 int *r_tot_chars)
-{
-  int missing = 0;
-  size_t i = 0;
-
-  *r_tot_chars = 0;
-  while (i < len) {
-    unsigned int c;
+/* -------------------------------------------------------------------- */
+/** \name Font Subsystem Init/Exit
+ * \{ */
 
-    if ((c = str[i]) < GLYPH_ASCII_TABLE_SIZE) {
-      i++;
-    }
-    else if ((c = BLI_str_utf8_as_unicode_step(str, &i)) != BLI_UTF8_ERR) {
-      if (FT_Get_Char_Index((font)->face, c) == 0) {
-        missing++;
-      }
-    }
-    (*r_tot_chars)++;
-  }
-  return missing;
+int blf_font_init(void)
+{
+  memset(&g_batch, 0, sizeof(g_batch));
+  BLI_spin_init(&ft_lib_mutex);
+  BLI_spin_init(&blf_glyph_cache_mutex);
+  return FT_Init_FreeType(&ft_lib);
 }
 
-void blf_font_free(FontBLF *font)
+void blf_font_exit(void)
 {
-  BLI_spin_lock(&blf_glyph_cache_mutex);
-  GlyphCacheBLF *gc;
-
-  while ((gc = BLI_pophead(&font->cache))) {
-    blf_glyph_cache_free(gc);
-  }
-
-  if (font->kerning_cache) {
-    MEM_freeN(font->kerning_cache);
-  }
+  FT_Done_FreeType(ft_lib);
+  BLI_spin_end(&ft_lib_mutex);
+  BLI_spin_end(&blf_glyph_cache_mutex);
+  blf_batch_draw_exit();
+}
 
-  FT_Done_Face(font->face);
-  if (font->filename) {
-    MEM_freeN(font->filename);
-  }
-  if (font->name) {
-    MEM_freeN(font->name);
-  }
-  MEM_freeN(font);
+/** \} */
 
-  BLI_spin_unlock(&blf_glyph_cache_mutex);
-}
+/* -------------------------------------------------------------------- */
+/** \name Font New/Free
+ * \{ */
 
 static void blf_font_fill(FontBLF *font)
 {
@@ -1366,54 +1414,69 @@ FontBLF *blf_font_new_from_mem(const char *name, const unsigned char *mem, int m
   return font;
 }
 
-int blf_font_height_max(FontBLF *font)
+void blf_font_free(FontBLF *font)
 {
-  int height_max;
-
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  height_max = gc->glyph_height_max;
+  BLI_spin_lock(&blf_glyph_cache_mutex);
+  GlyphCacheBLF *gc;
 
-  blf_glyph_cache_release(font);
-  return height_max;
-}
+  while ((gc = BLI_pophead(&font->cache))) {
+    blf_glyph_cache_free(gc);
+  }
 
-int blf_font_width_max(FontBLF *font)
-{
-  int width_max;
+  if (font->kerning_cache) {
+    MEM_freeN(font->kerning_cache);
+  }
 
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  width_max = gc->glyph_width_max;
+  FT_Done_Face(font->face);
+  if (font->filename) {
+    MEM_freeN(font->filename);
+  }
+  if (font->name) {
+    MEM_freeN(font->name);
+  }
+  MEM_freeN(font);
 
-  blf_glyph_cache_release(font);
-  return width_max;
+  BLI_spin_unlock(&blf_glyph_cache_mutex);
 }
 
-float blf_font_descender(FontBLF *font)
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Font Configure
+ * \{ */
+
+void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi)
 {
-  float descender;
+  GlyphCacheBLF *gc;
+  FT_Error err;
 
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  descender = gc->descender;
+  blf_glyph_cache_acquire(font);
 
-  blf_glyph_cache_release(font);
-  return descender;
-}
+  gc = blf_glyph_cache_find(font, size, dpi);
+  if (gc) {
+    /* Optimization: do not call FT_Set_Char_Size if size did not change. */
+    if (font->size == size && font->dpi == dpi) {
+      blf_glyph_cache_release(font);
+      return;
+    }
+  }
 
-float blf_font_ascender(FontBLF *font)
-{
-  float ascender;
+  err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi);
+  if (err) {
+    /* FIXME: here we can go through the fixed size and choice a close one */
+    printf("The current font don't support the size, %u and dpi, %u\n", size, dpi);
 
-  GlyphCacheBLF *gc = blf_glyph_cache_acquire(font);
-  ascender = gc->ascender;
+    blf_glyph_cache_release(font);
+    return;
+  }
 
-  blf_glyph_cache_release(font);
-  return ascender;
-}
+  font->size = size;
+  font->dpi = dpi;
 
-char *blf_display_name(FontBLF *font)
-{
-  if (!font->face->family_name) {
-    return NULL;
+  if (!gc) {
+    blf_glyph_cache_new(font);
   }
-  return BLI_sprintfN("%s %s", font->face->family_name, font->face->style_name);
+  blf_glyph_cache_release(font);
 }
+
+/** \} */
-- 
cgit v1.2.3


From 0b7947e85528bbf5c4e1f41354a41dc113937088 Mon Sep 17 00:00:00 2001
From: Campbell Barton <ideasman42@gmail.com>
Date: Sat, 21 Aug 2021 17:44:25 +1000
Subject: Cleanup: minor changes to blf_font.c

- Use early return when kerning isn't used.
- Remove early return that prevented matching acquire/release calls.
---
 source/blender/blenfont/intern/blf_font.c | 78 ++++++++++++++-----------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c
index 47fa3544474..75a2e893119 100644
--- a/source/blender/blenfont/intern/blf_font.c
+++ b/source/blender/blenfont/intern/blf_font.c
@@ -328,29 +328,31 @@ BLI_INLINE void blf_kerning_step_fast(FontBLF *font,
                                       const uint c,
                                       int *pen_x_p)
 {
-  if (FT_HAS_KERNING(font->face) && g_prev != NULL) {
-    FT_Vector delta = {KERNING_ENTRY_UNSET};
+  if (!FT_HAS_KERNING(font->face) || g_prev == NULL) {
+    return;
+  }
 
-    /* Get unscaled kerning value from our cache if ASCII. */
-    if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
-      delta.x = font->kerning_cache->ascii_table[c][c_prev];
-    }
+  FT_Vector delta = {KERNING_ENTRY_UNSET};
 
-    /* If not ASCII or not found in cache, ask FreeType for kerning. */
-    if (UNLIKELY(delta.x == KERNING_ENTRY_UNSET)) {
-      /* Note that this function sets delta values to zero on any error. */
-      FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_UNSCALED, &delta);
-    }
+  /* Get unscaled kerning value from our cache if ASCII. */
+  if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
+    delta.x = font->kerning_cache->ascii_table[c][c_prev];
+  }
 
-    /* If ASCII we save this value to our cache for quicker access next time. */
-    if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
-      font->kerning_cache->ascii_table[c][c_prev] = (int)delta.x;
-    }
+  /* If not ASCII or not found in cache, ask FreeType for kerning. */
+  if (UNLIKELY(delta.x == KERNING_ENTRY_UNSET)) {
+    /* Note that this function sets delta values to zero on any error. */
+    FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_UNSCALED, &delta);
+  }
 
-    if (delta.x != 0) {
-      /* Convert unscaled design units to pixels and move pen. */
-      *pen_x_p += blf_unscaled_F26Dot6_to_pixels(font, delta.x);
-    }
+  /* If ASCII we save this value to our cache for quicker access next time. */
+  if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) {
+    font->kerning_cache->ascii_table[c][c_prev] = (int)delta.x;
+  }
+
+  if (delta.x != 0) {
+    /* Convert unscaled design units to pixels and move pen. */
+    *pen_x_p += blf_unscaled_F26Dot6_to_pixels(font, delta.x);
   }
 }
 
@@ -1447,35 +1449,27 @@ void blf_font_free(FontBLF *font)
 
 void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi)
 {
-  GlyphCacheBLF *gc;
-  FT_Error err;
-
   blf_glyph_cache_acquire(font);
 
-  gc = blf_glyph_cache_find(font, size, dpi);
-  if (gc) {
+  GlyphCacheBLF *gc = blf_glyph_cache_find(font, size, dpi);
+  if (gc && (font->size == size && font->dpi == dpi)) {
     /* Optimization: do not call FT_Set_Char_Size if size did not change. */
-    if (font->size == size && font->dpi == dpi) {
-      blf_glyph_cache_release(font);
-      return;
-    }
   }
-
-  err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi);
-  if (err) {
-    /* FIXME: here we can go through the fixed size and choice a close one */
-    printf("The current font don't support the size, %u and dpi, %u\n", size, dpi);
-
-    blf_glyph_cache_release(font);
-    return;
+  else {
+    const FT_Error err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi);
+    if (err) {
+      /* FIXME: here we can go through the fixed size and choice a close one */
+      printf("The current font don't support the size, %u and dpi, %u\n", size, dpi);
+    }
+    else {
+      font->size = size;
+      font->dpi = dpi;
+      if (gc == NULL) {
+        blf_glyph_cache_new(font);
+      }
+    }
   }
 
-  font->size = size;
-  font->dpi = dpi;
-
-  if (!gc) {
-    blf_glyph_cache_new(font);
-  }
   blf_glyph_cache_release(font);
 }
 
-- 
cgit v1.2.3


From 34a05f39be2b79dd1c508c374a47cee6792174f9 Mon Sep 17 00:00:00 2001
From: Ankit Meel <ankitjmeel@gmail.com>
Date: Sat, 21 Aug 2021 14:02:50 +0530
Subject: Clang: warn about C++20 designated initializers

With the ongoing transition to C++ files, Windows build
breaks often because of designated initializers.
Now we have two compilers to catch the MSVC build error on.

Reviewed By: #platform_macos, brecht, campbellbarton
Differential Revision: https://developer.blender.org/D11940
---
 CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2868324bf46..5a555876d21 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1598,6 +1598,10 @@ elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
   ADD_CHECK_C_COMPILER_FLAG(C_WARNINGS C_WARN_UNUSED_PARAMETER -Wunused-parameter)
 
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_ALL -Wall)
+  # Designated initializer is a C++20 feature & breaks MSVC build. Dropping MSVC 2019 or
+  # updating to C++20 allows removing this.
+  ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_CXX20_DESIGNATOR -Wc++20-designator)
+
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_AUTOLOGICAL_COMPARE -Wno-tautological-compare)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_UNKNOWN_PRAGMAS     -Wno-unknown-pragmas)
   ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_CHAR_SUBSCRIPTS     -Wno-char-subscripts)
-- 
cgit v1.2.3


From 2b170f16d6ded9b3bcb428121b27274ae8637555 Mon Sep 17 00:00:00 2001
From: Lukas Stockner <lukas.stockner@freenet.de>
Date: Thu, 19 Aug 2021 23:57:00 +0200
Subject: Refactor low-level blendfile reading into separate files

Instead of handling mmap, compression etc. all directly in readfile.c, refactor
the code to use a generic FileReader.
This makes it easier to add new compression methods or similar, and allows to
reuse the logic in other places (e.g. thumbnail reading).

Reviewed By: campbellbarton, brecht, mont29

Differential Revision: https://developer.blender.org/D5799
---
 source/blender/blenlib/BLI_fileops.h               |   2 +
 source/blender/blenlib/BLI_filereader.h            |  79 ++++
 source/blender/blenlib/CMakeLists.txt              |   4 +
 source/blender/blenlib/intern/fileops.c            |   7 +
 source/blender/blenlib/intern/filereader_file.c    |  80 ++++
 source/blender/blenlib/intern/filereader_gzip.c    | 108 +++++
 source/blender/blenlib/intern/filereader_memory.c  | 145 +++++++
 source/blender/blenloader/BLO_undofile.h           |  14 +
 source/blender/blenloader/intern/readfile.c        | 449 ++++-----------------
 source/blender/blenloader/intern/readfile.h        |  32 +-
 source/blender/blenloader/intern/undofile.c        |  95 +++++
 source/blender/blenloader/intern/versioning_250.c  |   3 +-
 .../blender/blenloader/intern/versioning_legacy.c  |   3 +-
 source/blender/blenloader/intern/writefile.c       |   2 +-
 source/blender/windowmanager/intern/wm_files.c     |  88 ++--
 15 files changed, 667 insertions(+), 444 deletions(-)
 create mode 100644 source/blender/blenlib/BLI_filereader.h
 create mode 100644 source/blender/blenlib/intern/filereader_file.c
 create mode 100644 source/blender/blenlib/intern/filereader_gzip.c
 create mode 100644 source/blender/blenlib/intern/filereader_memory.c

diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 7cfecc798a7..12fa73279c8 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -166,6 +166,8 @@ size_t BLI_gzip_mem_to_file_at_pos(void *buf,
                                    int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
     ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+bool BLI_file_magic_is_gzip(const char header[4]);
+
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
 size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 
diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h
new file mode 100644
index 00000000000..dbcaa7cb1b6
--- /dev/null
+++ b/source/blender/blenlib/BLI_filereader.h
@@ -0,0 +1,79 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ * \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
+ */
+
+#pragma once
+
+#ifdef WIN32
+#  include "BLI_winstuff.h"
+#else
+#  include <sys/types.h>
+#endif
+
+#include "BLI_compiler_attrs.h"
+#include "BLI_utildefines.h"
+
+#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
+typedef int64_t off64_t;
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct FileReader;
+
+typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
+typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
+typedef void (*FileReaderCloseFn)(struct FileReader *reader);
+
+/* General structure for all FileReaders, implementations add custom fields at the end. */
+typedef struct FileReader {
+  FileReaderReadFn read;
+  FileReaderSeekFn seek;
+  FileReaderCloseFn close;
+
+  off64_t offset;
+} FileReader;
+
+/* Functions for opening the various types of FileReader.
+ * They either succeed and return a valid FileReader, or fail and return NULL.
+ *
+ * If a FileReader is created, it has to be cleaned up and freed by calling
+ * its close() function unless another FileReader has taken ownership - for example,
+ * Gzip takes over the base FileReader and will clean it up when their clean() is called.
+ */
+
+/* Create FileReader from raw file descriptor. */
+FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from raw file descriptor using memory-mapped IO. */
+FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
+/* Create FileReader from a region of memory. */
+FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
+    ATTR_NONNULL();
+/* Create FileReader from applying Gzip decompression on an underlying file. */
+FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index b6603dce378..d2ba9e74c90 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -75,6 +75,9 @@ set(SRC
   intern/endian_switch.c
   intern/expr_pylike_eval.c
   intern/fileops.c
+  intern/filereader_file.c
+  intern/filereader_gzip.c
+  intern/filereader_memory.c
   intern/fnmatch.c
   intern/freetypefont.c
   intern/gsqueue.c
@@ -194,6 +197,7 @@ set(SRC
   BLI_enumerable_thread_specific.hh
   BLI_expr_pylike_eval.h
   BLI_fileops.h
+  BLI_filereader.h
   BLI_fileops_types.h
   BLI_float2.hh
   BLI_float3.hh
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index ac034d2b5cd..6fc2222241b 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
 
 #undef CHUNK
 
+bool BLI_file_magic_is_gzip(const char header[4])
+{
+  /* GZIP itself starts with the magic bytes 0x1f 0x8b.
+   * The third byte indicates the compression method, which is 0x08 for DEFLATE. */
+  return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
+}
+
 /**
  * Returns true if the file with the specified name can be written.
  * This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c
new file mode 100644
index 00000000000..3a833871e27
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_file.c
@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#ifndef WIN32
+#  include <unistd.h> /* for read close */
+#else
+#  include "BLI_winstuff.h"
+#  include "winsock2.h"
+#  include <io.h> /* for open close read */
+#endif
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  int filedes;
+} RawFileReader;
+
+static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  ssize_t readsize = read(rawfile->filedes, buffer, size);
+
+  if (readsize >= 0) {
+    rawfile->reader.offset += readsize;
+  }
+
+  return readsize;
+}
+
+static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
+  return rawfile->reader.offset;
+}
+
+static void file_close(FileReader *reader)
+{
+  RawFileReader *rawfile = (RawFileReader *)reader;
+  close(rawfile->filedes);
+  MEM_freeN(rawfile);
+}
+
+FileReader *BLI_filereader_new_file(int filedes)
+{
+  RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
+
+  rawfile->filedes = filedes;
+
+  rawfile->reader.read = file_read;
+  rawfile->reader.seek = file_seek;
+  rawfile->reader.close = file_close;
+
+  return (FileReader *)rawfile;
+}
diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c
new file mode 100644
index 00000000000..72eb153a8b9
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_gzip.c
@@ -0,0 +1,108 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <zlib.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  FileReader *base;
+
+  z_stream strm;
+
+  void *in_buf;
+  size_t in_size;
+} GzipReader;
+
+static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  gzip->strm.avail_out = size;
+  gzip->strm.next_out = buffer;
+
+  while (gzip->strm.avail_out > 0) {
+    if (gzip->strm.avail_in == 0) {
+      /* Ran out of buffered input data, read some more. */
+      size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
+
+      if (readsize > 0) {
+        /* We got some data, so mark the buffer as refilled. */
+        gzip->strm.avail_in = readsize;
+        gzip->strm.next_in = gzip->in_buf;
+      }
+      else {
+        /* The underlying file is EOF, so return as much as we can. */
+        break;
+      }
+    }
+
+    int ret = inflate(&gzip->strm, Z_NO_FLUSH);
+
+    if (ret != Z_OK && ret != Z_BUF_ERROR) {
+      break;
+    }
+  }
+
+  ssize_t read_len = size - gzip->strm.avail_out;
+  gzip->reader.offset += read_len;
+  return read_len;
+}
+
+static void gzip_close(FileReader *reader)
+{
+  GzipReader *gzip = (GzipReader *)reader;
+
+  if (inflateEnd(&gzip->strm) != Z_OK) {
+    printf("close gzip stream error\n");
+  }
+  MEM_freeN((void *)gzip->in_buf);
+
+  gzip->base->close(gzip->base);
+  MEM_freeN(gzip);
+}
+
+FileReader *BLI_filereader_new_gzip(FileReader *base)
+{
+  GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
+  gzip->base = base;
+
+  if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
+    MEM_freeN(gzip);
+    return NULL;
+  }
+
+  gzip->in_size = 256 * 2014;
+  gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
+
+  gzip->reader.read = gzip_read;
+  gzip->reader.seek = NULL;
+  gzip->reader.close = gzip_close;
+
+  return (FileReader *)gzip;
+}
diff --git a/source/blender/blenlib/intern/filereader_memory.c b/source/blender/blenlib/intern/filereader_memory.c
new file mode 100644
index 00000000000..150fed7d1cc
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_memory.c
@@ -0,0 +1,145 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2004-2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_filereader.h"
+#include "BLI_mmap.h"
+
+#include "MEM_guardedalloc.h"
+
+/* This file implements both memory-backed and memory-mapped-file-backed reading. */
+typedef struct {
+  FileReader reader;
+
+  const char *data;
+  BLI_mmap_file *mmap;
+  size_t length;
+} MemoryReader;
+
+static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  memcpy(buffer, mem->data + mem->reader.offset, readsize);
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  off64_t new_pos;
+  if (whence == SEEK_CUR) {
+    new_pos = mem->reader.offset + offset;
+  }
+  else if (whence == SEEK_SET) {
+    new_pos = offset;
+  }
+  else if (whence == SEEK_END) {
+    new_pos = mem->length + offset;
+  }
+  else {
+    return -1;
+  }
+
+  if (new_pos < 0 || new_pos > mem->length) {
+    return -1;
+  }
+
+  mem->reader.offset = new_pos;
+  return mem->reader.offset;
+}
+
+static void memory_close_raw(FileReader *reader)
+{
+  MEM_freeN(reader);
+}
+
+FileReader *BLI_filereader_new_memory(const void *data, size_t len)
+{
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->data = (const char *)data;
+  mem->length = len;
+
+  mem->reader.read = memory_read_raw;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_raw;
+
+  return (FileReader *)mem;
+}
+
+/* Memory-mapped file reading.
+ * By using `mmap()`, we can map a file so that it can be treated like normal memory,
+ * meaning that we can just read from it with `memcpy()` etc.
+ * This avoids system call overhead and can significantly speed up file loading.
+ */
+
+static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+
+  /* Don't read more bytes than there are available in the buffer. */
+  size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
+
+  if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
+    return 0;
+  }
+
+  mem->reader.offset += readsize;
+
+  return readsize;
+}
+
+static void memory_close_mmap(FileReader *reader)
+{
+  MemoryReader *mem = (MemoryReader *)reader;
+  BLI_mmap_free(mem->mmap);
+  MEM_freeN(mem);
+}
+
+FileReader *BLI_filereader_new_mmap(int filedes)
+{
+  BLI_mmap_file *mmap = BLI_mmap_open(filedes);
+  if (mmap == NULL) {
+    return NULL;
+  }
+
+  MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
+
+  mem->mmap = mmap;
+  mem->length = BLI_lseek(filedes, 0, SEEK_END);
+
+  mem->reader.read = memory_read_mmap;
+  mem->reader.seek = memory_seek;
+  mem->reader.close = memory_close_mmap;
+
+  return (FileReader *)mem;
+}
diff --git a/source/blender/blenloader/BLO_undofile.h b/source/blender/blenloader/BLO_undofile.h
index fc41a6e832f..4e240e2462b 100644
--- a/source/blender/blenloader/BLO_undofile.h
+++ b/source/blender/blenloader/BLO_undofile.h
@@ -24,6 +24,8 @@
  * \ingroup blenloader
  */
 
+#include "BLI_filereader.h"
+
 struct GHash;
 struct Scene;
 
@@ -65,6 +67,16 @@ typedef struct MemFileUndoData {
   size_t undo_size;
 } MemFileUndoData;
 
+/* FileReader-compatible wrapper for reading MemFiles */
+typedef struct {
+  FileReader reader;
+
+  MemFile *memfile;
+  int undo_direction;
+
+  bool memchunk_identical;
+} UndoReader;
+
 /* actually only used writefile.c */
 
 void BLO_memfile_write_init(MemFileWriteData *mem_data,
@@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile,
                                          struct Main *bmain,
                                          struct Scene **r_scene);
 extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename);
+
+FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction);
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index e48c305fc4b..82fbb2083df 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -21,8 +21,6 @@
  * \ingroup blenloader
  */
 
-#include "zlib.h"
-
 #include <ctype.h> /* for isdigit. */
 #include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
 #include <limits.h>
@@ -71,7 +69,6 @@
 #include "BLI_math.h"
 #include "BLI_memarena.h"
 #include "BLI_mempool.h"
-#include "BLI_mmap.h"
 #include "BLI_threads.h"
 
 #include "PIL_time.h"
@@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd)
        */
       if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
         bhead4.code = DATA;
-        readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
+        readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4));
 
         if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
           if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd)
       }
       else {
         bhead8.code = DATA;
-        readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
+        readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8));
 
         if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
           if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd)
         /* pass */
       }
 #ifdef USE_BHEAD_READ_ON_DEMAND
-      else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
+      else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
         /* Delay reading bhead content. */
         new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead");
         if (new_bhead) {
           new_bhead->next = new_bhead->prev = NULL;
-          new_bhead->file_offset = fd->file_offset;
+          new_bhead->file_offset = fd->file->offset;
           new_bhead->has_data = false;
           new_bhead->is_memchunk_identical = false;
           new_bhead->bhead = bhead;
-          off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
+          off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR);
           if (seek_new == -1) {
             fd->is_eof = true;
             MEM_freeN(new_bhead);
             new_bhead = NULL;
           }
-          BLI_assert(fd->file_offset == seek_new);
+          BLI_assert(fd->file->offset == seek_new);
         }
         else {
           fd->is_eof = true;
@@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd)
           new_bhead->is_memchunk_identical = false;
           new_bhead->bhead = bhead;
 
-          readsize = fd->read(
-              fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
+          readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len);
 
-          if (readsize != (ssize_t)bhead.len) {
+          if (readsize != bhead.len) {
             fd->is_eof = true;
             MEM_freeN(new_bhead);
             new_bhead = NULL;
           }
+
+          if (fd->flags & FD_FLAGS_IS_MEMFILE) {
+            new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
+          }
         }
         else {
           fd->is_eof = true;
@@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
   bool success = true;
   BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
   BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
-  off64_t offset_backup = fd->file_offset;
-  if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
+  off64_t offset_backup = fd->file->offset;
+  if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) {
     success = false;
   }
   else {
-    if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
-        (ssize_t)new_bhead->bhead.len) {
+    if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) {
       success = false;
     }
+    if (fd->flags & FD_FLAGS_IS_MEMFILE) {
+      new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
+    }
   }
-  if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
+  if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) {
     success = false;
   }
   return success;
@@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd)
   ssize_t readsize;
 
   /* read in the header data */
-  readsize = fd->read(fd, header, sizeof(header), NULL);
+  readsize = fd->file->read(fd->file, header, sizeof(header));
 
   if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
       ELEM(header[8], 'v', 'V') &&
@@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd)
 
 /** \} */
 
-/* -------------------------------------------------------------------- */
-/** \name File Data API
- * \{ */
-
-/* Regular file reading. */
-
-static ssize_t fd_read_data_from_file(FileData *filedata,
-                                      void *buffer,
-                                      size_t size,
-                                      bool *UNUSED(r_is_memchunck_identical))
-{
-  ssize_t readsize = read(filedata->filedes, buffer, size);
-
-  if (readsize < 0) {
-    readsize = EOF;
-  }
-  else {
-    filedata->file_offset += readsize;
-  }
-
-  return readsize;
-}
-
-static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
-{
-  filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
-  return filedata->file_offset;
-}
-
-/* GZip file reading. */
-
-static ssize_t fd_read_gzip_from_file(FileData *filedata,
-                                      void *buffer,
-                                      size_t size,
-                                      bool *UNUSED(r_is_memchunck_identical))
-{
-  BLI_assert(size <= INT_MAX);
-
-  ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
-
-  if (readsize < 0) {
-    readsize = EOF;
-  }
-  else {
-    filedata->file_offset += readsize;
-  }
-
-  return readsize;
-}
-
-/* Memory reading. */
-
-static ssize_t fd_read_from_memory(FileData *filedata,
-                                   void *buffer,
-                                   size_t size,
-                                   bool *UNUSED(r_is_memchunck_identical))
-{
-  /* don't read more bytes than there are available in the buffer */
-  ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
-
-  memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
-  filedata->file_offset += readsize;
-
-  return readsize;
-}
-
-/* Memory-mapped file reading.
- * By using mmap(), we can map a file so that it can be treated like normal memory,
- * meaning that we can just read from it with memcpy() etc.
- * This avoids system call overhead and can significantly speed up file loading.
- */
-
-static ssize_t fd_read_from_mmap(FileData *filedata,
-                                 void *buffer,
-                                 size_t size,
-                                 bool *UNUSED(r_is_memchunck_identical))
-{
-  /* don't read more bytes than there are available in the buffer */
-  size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset));
-
-  if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) {
-    return 0;
-  }
-
-  filedata->file_offset += readsize;
-
-  return readsize;
-}
-
-static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence)
-{
-  off64_t new_pos;
-  if (whence == SEEK_CUR) {
-    new_pos = filedata->file_offset + offset;
-  }
-  else if (whence == SEEK_SET) {
-    new_pos = offset;
-  }
-  else if (whence == SEEK_END) {
-    new_pos = filedata->buffersize + offset;
-  }
-  else {
-    return -1;
-  }
-
-  if (new_pos < 0 || new_pos > filedata->buffersize) {
-    return -1;
-  }
-
-  filedata->file_offset = new_pos;
-  return filedata->file_offset;
-}
-
-/* MemFile reading. */
-
-static ssize_t fd_read_from_memfile(FileData *filedata,
-                                    void *buffer,
-                                    size_t size,
-                                    bool *r_is_memchunck_identical)
-{
-  static size_t seek = SIZE_MAX; /* the current position */
-  static size_t offset = 0;      /* size of previous chunks */
-  static MemFileChunk *chunk = NULL;
-  size_t chunkoffset, readsize, totread;
-
-  if (r_is_memchunck_identical != NULL) {
-    *r_is_memchunck_identical = true;
-  }
-
-  if (size == 0) {
-    return 0;
-  }
-
-  if (seek != (size_t)filedata->file_offset) {
-    chunk = filedata->memfile->chunks.first;
-    seek = 0;
-
-    while (chunk) {
-      if (seek + chunk->size > (size_t)filedata->file_offset) {
-        break;
-      }
-      seek += chunk->size;
-      chunk = chunk->next;
-    }
-    offset = seek;
-    seek = (size_t)filedata->file_offset;
-  }
-
-  if (chunk) {
-    totread = 0;
-
-    do {
-      /* first check if it's on the end if current chunk */
-      if (seek - offset == chunk->size) {
-        offset += chunk->size;
-        chunk = chunk->next;
-      }
-
-      /* debug, should never happen */
-      if (chunk == NULL) {
-        CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk");
-        return 0;
-      }
-
-      chunkoffset = seek - offset;
-      readsize = size - totread;
-
-      /* data can be spread over multiple chunks, so clamp size
-       * to within this chunk, and then it will read further in
-       * the next chunk */
-      if (chunkoffset + readsize > chunk->size) {
-        readsize = chunk->size - chunkoffset;
-      }
-
-      memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
-      totread += readsize;
-      filedata->file_offset += readsize;
-      seek += readsize;
-      if (r_is_memchunck_identical != NULL) {
-        /* `is_identical` of current chunk represents whether it changed compared to previous undo
-         * step. this is fine in redo case, but not in undo case, where we need an extra flag
-         * defined when saving the next (future) step after the one we want to restore, as we are
-         * supposed to 'come from' that future undo step, and not the one before current one. */
-        *r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ?
-                                         chunk->is_identical :
-                                         chunk->is_identical_future;
-      }
-    } while (totread < size);
-
-    return (ssize_t)totread;
-  }
-
-  return 0;
-}
-
 static FileData *filedata_new(BlendFileReadReport *reports)
 {
   BLI_assert(reports != NULL);
 
   FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
 
-  fd->filedes = -1;
-  fd->gzfiledes = NULL;
-
   fd->memsdna = DNA_sdna_current_get();
 
   fd->datamap = oldnewmap_new();
@@ -1387,78 +1191,60 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
 
 static FileData *blo_filedata_from_file_descriptor(const char *filepath,
                                                    BlendFileReadReport *reports,
-                                                   int file)
+                                                   int filedes)
 {
-  FileDataReadFn *read_fn = NULL;
-  FileDataSeekFn *seek_fn = NULL; /* Optional. */
-  size_t buffersize = 0;
-  BLI_mmap_file *mmap_file = NULL;
-
-  gzFile gzfile = (gzFile)Z_NULL;
-
   char header[7];
+  FileReader *rawfile = BLI_filereader_new_file(filedes);
+  FileReader *file = NULL;
 
-  /* Regular file. */
   errno = 0;
-  if (read(file, header, sizeof(header)) != sizeof(header)) {
+  /* If opening the file failed or we can't read the header, give up. */
+  if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
     BKE_reportf(reports->reports,
                 RPT_WARNING,
                 "Unable to read '%s': %s",
                 filepath,
                 errno ? strerror(errno) : TIP_("insufficient content"));
+    if (rawfile) {
+      rawfile->close(rawfile);
+    }
+    else {
+      close(filedes);
+    }
     return NULL;
   }
 
-  /* Regular file. */
-  if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
-    read_fn = fd_read_data_from_file;
-    seek_fn = fd_seek_data_from_file;
+  /* Rewind the file after reading the header. */
+  rawfile->seek(rawfile, 0, SEEK_SET);
 
-    mmap_file = BLI_mmap_open(file);
-    if (mmap_file != NULL) {
-      read_fn = fd_read_from_mmap;
-      seek_fn = fd_seek_from_mmap;
-      buffersize = BLI_lseek(file, 0, SEEK_END);
+  /* Check if we have a regular file. */
+  if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
+    /* Try opening the file with memory-mapped IO. */
+    file = BLI_filereader_new_mmap(filedes);
+    if (file == NULL) {
+      /* mmap failed, so just keep using rawfile. */
+      file = rawfile;
+      rawfile = NULL;
     }
   }
-
-  BLI_lseek(file, 0, SEEK_SET);
-
-  /* Gzip file. */
-  errno = 0;
-  if ((read_fn == NULL) &&
-      /* Check header magic. */
-      (header[0] == 0x1f && header[1] == 0x8b)) {
-    gzfile = BLI_gzopen(filepath, "rb");
-    if (gzfile == (gzFile)Z_NULL) {
-      BKE_reportf(reports->reports,
-                  RPT_WARNING,
-                  "Unable to open '%s': %s",
-                  filepath,
-                  errno ? strerror(errno) : TIP_("unknown error reading file"));
-      return NULL;
+  else if (BLI_file_magic_is_gzip(header)) {
+    file = BLI_filereader_new_gzip(rawfile);
+    if (file != NULL) {
+      rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
     }
-
-    /* 'seek_fn' is too slow for gzip, don't set it. */
-    read_fn = fd_read_gzip_from_file;
-    /* Caller must close. */
-    file = -1;
   }
 
-  if (read_fn == NULL) {
+  /* Clean up `rawfile` if it wasn't taken over. */
+  if (rawfile != NULL) {
+    rawfile->close(rawfile);
+  }
+  if (file == NULL) {
     BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath);
     return NULL;
   }
 
   FileData *fd = filedata_new(reports);
-
-  fd->filedes = file;
-  fd->gzfiledes = gzfile;
-
-  fd->read = read_fn;
-  fd->seek = seek_fn;
-  fd->mmap_file = mmap_file;
-  fd->buffersize = buffersize;
+  fd->file = file;
 
   return fd;
 }
@@ -1475,11 +1261,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead
                 errno ? strerror(errno) : TIP_("unknown error reading file"));
     return NULL;
   }
-  FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file);
-  if ((fd == NULL) || (fd->filedes == -1)) {
-    close(file);
-  }
-  return fd;
+  return blo_filedata_from_file_descriptor(filepath, reports, file);
 }
 
 /* cannot be called with relative paths anymore! */
@@ -1513,50 +1295,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
   return NULL;
 }
 
-static ssize_t fd_read_gzip_from_memory(FileData *filedata,
-                                        void *buffer,
-                                        size_t size,
-                                        bool *UNUSED(r_is_memchunck_identical))
-{
-  int err;
-
-  filedata->strm.next_out = (Bytef *)buffer;
-  filedata->strm.avail_out = (uint)size;
-
-  /* Inflate another chunk. */
-  err = inflate(&filedata->strm, Z_SYNC_FLUSH);
-
-  if (err == Z_STREAM_END) {
-    return 0;
-  }
-  if (err != Z_OK) {
-    CLOG_ERROR(&LOG, "ZLib error (code %d)", err);
-    return 0;
-  }
-
-  filedata->file_offset += size;
-
-  return (ssize_t)size;
-}
-
-static int fd_read_gzip_from_memory_init(FileData *fd)
-{
-
-  fd->strm.next_in = (Bytef *)fd->buffer;
-  fd->strm.avail_in = fd->buffersize;
-  fd->strm.total_out = 0;
-  fd->strm.zalloc = Z_NULL;
-  fd->strm.zfree = Z_NULL;
-
-  if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) {
-    return 0;
-  }
-
-  fd->read = fd_read_gzip_from_memory;
-
-  return 1;
-}
-
 FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports)
 {
   if (!mem || memsize < SIZEOFBLENDERHEADER) {
@@ -1565,24 +1303,21 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
     return NULL;
   }
 
-  FileData *fd = filedata_new(reports);
-  const char *cp = mem;
-
-  fd->buffer = mem;
-  fd->buffersize = memsize;
+  FileReader *mem_file = BLI_filereader_new_memory(mem, memsize);
+  FileReader *file = mem_file;
 
-  /* test if gzip */
-  if (cp[0] == 0x1f && cp[1] == 0x8b) {
-    if (0 == fd_read_gzip_from_memory_init(fd)) {
-      blo_filedata_free(fd);
-      return NULL;
-    }
+  if (BLI_file_magic_is_gzip(mem)) {
+    file = BLI_filereader_new_gzip(mem_file);
   }
-  else {
-    fd->read = fd_read_from_memory;
+
+  if (file == NULL) {
+    /* Compression initialization failed. */
+    mem_file->close(mem_file);
+    return NULL;
   }
 
-  fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
+  FileData *fd = filedata_new(reports);
+  fd->file = file;
 
   return blo_decode_and_check(fd, reports->reports);
 }
@@ -1597,11 +1332,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
   }
 
   FileData *fd = filedata_new(reports);
-  fd->memfile = memfile;
+  fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction);
   fd->undo_direction = params->undo_direction;
-
-  fd->read = fd_read_from_memfile;
-  fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
+  fd->flags |= FD_FLAGS_IS_MEMFILE;
 
   return blo_decode_and_check(fd, reports->reports);
 }
@@ -1609,30 +1342,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
 void blo_filedata_free(FileData *fd)
 {
   if (fd) {
-    if (fd->filedes != -1) {
-      close(fd->filedes);
-    }
-
-    if (fd->gzfiledes != NULL) {
-      gzclose(fd->gzfiledes);
-    }
-
-    if (fd->strm.next_in) {
-      int err = inflateEnd(&fd->strm);
-      if (err != Z_OK) {
-        CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err);
-      }
-    }
-
-    if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
-      MEM_freeN((void *)fd->buffer);
-      fd->buffer = NULL;
-    }
-
-    if (fd->mmap_file) {
-      BLI_mmap_free(fd->mmap_file);
-      fd->mmap_file = NULL;
-    }
+    fd->file->close(fd->file);
 
     /* Free all BHeadN data blocks */
 #ifndef NDEBUG
@@ -1640,7 +1350,7 @@ void blo_filedata_free(FileData *fd)
 #else
     /* Sanity check we're not keeping memory we don't need. */
     LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) {
-      if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
+      if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
         BLI_assert(new_bhead->has_data == 0);
       }
       MEM_freeN(new_bhead);
@@ -2096,7 +1806,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id),
 
 void blo_cache_storage_init(FileData *fd, Main *bmain)
 {
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     BLI_assert(fd->cache_storage == NULL);
     fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__);
     fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
@@ -2261,7 +1971,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
  * undo since DNA must match. */
 static const void *peek_struct_undo(FileData *fd, BHead *bhead)
 {
-  BLI_assert(fd->memfile != NULL);
+  BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE);
   UNUSED_VARS_NDEBUG(fd);
   return (bhead->len) ? (const void *)(bhead + 1) : NULL;
 }
@@ -3679,7 +3389,7 @@ static BHead *read_libblock(FileData *fd,
    * When datablocks are changed but still exist, we restore them at the old
    * address and inherit recalc flags for the dependency graph. */
   ID *id_old = NULL;
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) {
       if (r_id) {
         *r_id = id_old;
@@ -3980,13 +3690,14 @@ static void lib_link_all(FileData *fd, Main *bmain)
       continue;
     }
 
-    if (fd->memfile != NULL && GS(id->name) == ID_WM) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) {
       /* No load UI for undo memfiles.
        * Only WM currently, SCR needs it still (see below), and so does WS? */
       continue;
     }
 
-    if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo &&
+        (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
       /* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across
        * current undo step, and old IDs re-use their old memory address, we do not need to liblink
        * it at all. */
@@ -4165,7 +3876,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
   BlendFileData *bfd;
   ListBase mainlist = {NULL, NULL};
 
-  if (fd->memfile != NULL) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step");
   }
 
@@ -4256,7 +3967,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
   }
 
   /* do before read_libraries, but skip undo case */
-  if (fd->memfile == NULL) {
+  if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
     if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
       do_versions(fd, NULL, bfd->main);
     }
@@ -4278,7 +3989,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
     fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
 
     /* Skip in undo case. */
-    if (fd->memfile == NULL) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
       /* Note that we can't recompute user-counts at this point in undo case, we play too much with
        * IDs from different memory realms, and Main database is not in a fully valid state yet.
        */
@@ -4311,7 +4022,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
 
     /* Now that all our data-blocks are loaded,
      * we can re-generate overrides from their references. */
-    if (fd->memfile == NULL) {
+    if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
       /* Do not apply in undo case! */
       fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
 
@@ -4391,7 +4102,7 @@ static void sort_bhead_old_map(FileData *fd)
 static BHead *find_previous_lib(FileData *fd, BHead *bhead)
 {
   /* Skip library data-blocks in undo, see comment in read_libblock. */
-  if (fd->memfile) {
+  if (fd->flags & FD_FLAGS_IS_MEMFILE) {
     return NULL;
   }
 
@@ -5850,7 +5561,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
 
 bool BLO_read_data_is_undo(BlendDataReader *reader)
 {
-  return reader->fd->memfile != NULL;
+  return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
 }
 
 void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr)
@@ -5870,7 +5581,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
 
 bool BLO_read_lib_is_undo(BlendLibReader *reader)
 {
-  return reader->fd->memfile != NULL;
+  return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
 }
 
 Main *BLO_read_lib_get_main(BlendLibReader *reader)
diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h
index b04043f9641..beeed8e45ae 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -28,10 +28,10 @@
 #  include "BLI_winstuff.h"
 #endif
 
+#include "BLI_filereader.h"
 #include "DNA_sdna_types.h"
 #include "DNA_space_types.h"
 #include "DNA_windowmanager_types.h" /* for ReportType */
-#include "zlib.h"
 
 struct BLI_mmap_file;
 struct BLOCacheStorage;
@@ -50,7 +50,7 @@ enum eFileDataFlag {
   FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
   FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2,
   FD_FLAGS_FILE_OK = 1 << 3,
-  FD_FLAGS_NOT_MY_BUFFER = 1 << 4,
+  FD_FLAGS_IS_MEMFILE = 1 << 4,
   /* XXX Unused in practice (checked once but never set). */
   FD_FLAGS_NOT_MY_LIBMAP = 1 << 5,
 };
@@ -60,44 +60,18 @@ enum eFileDataFlag {
 #  pragma GCC poison off_t
 #endif
 
-#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
-typedef int64_t off64_t;
-#endif
-
-typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
-                                void *buffer,
-                                size_t size,
-                                bool *r_is_memchunk_identical);
-typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
-
 typedef struct FileData {
   /** Linked list of BHeadN's. */
   ListBase bhead_list;
   enum eFileDataFlag flags;
   bool is_eof;
-  size_t buffersize;
-  off64_t file_offset;
 
-  FileDataReadFn *read;
-  FileDataSeekFn *seek;
+  FileReader *file;
 
-  /** Regular file reading. */
-  int filedes;
-
-  /** Variables needed for reading from memory / stream / memory-mapped files. */
-  const char *buffer;
-  struct BLI_mmap_file *mmap_file;
-  /** Variables needed for reading from memfile (undo). */
-  struct MemFile *memfile;
   /** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
    * to detect unchanged data from memfile. */
   int undo_direction; /* eUndoStepDir */
 
-  /** Variables needed for reading from file. */
-  gzFile gzfiledes;
-  /** Gzip stream for memory decompression. */
-  z_stream strm;
-
   /** Now only in use for library appending. */
   char relabase[FILE_MAX];
 
diff --git a/source/blender/blenloader/intern/undofile.c b/source/blender/blenloader/intern/undofile.c
index 2eeeac2e8d7..62072cf7df5 100644
--- a/source/blender/blenloader/intern/undofile.c
+++ b/source/blender/blenloader/intern/undofile.c
@@ -48,6 +48,7 @@
 
 #include "BKE_lib_id.h"
 #include "BKE_main.h"
+#include "BKE_undo_system.h"
 
 /* keep last */
 #include "BLI_strict_flags.h"
@@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename)
   }
   return true;
 }
+
+static ssize_t undo_read(FileReader *reader, void *buffer, size_t size)
+{
+  UndoReader *undo = (UndoReader *)reader;
+
+  static size_t seek = SIZE_MAX; /* The current position. */
+  static size_t offset = 0;      /* Size of previous chunks. */
+  static MemFileChunk *chunk = NULL;
+  size_t chunkoffset, readsize, totread;
+
+  undo->memchunk_identical = true;
+
+  if (size == 0) {
+    return 0;
+  }
+
+  if (seek != (size_t)undo->reader.offset) {
+    chunk = undo->memfile->chunks.first;
+    seek = 0;
+
+    while (chunk) {
+      if (seek + chunk->size > (size_t)undo->reader.offset) {
+        break;
+      }
+      seek += chunk->size;
+      chunk = chunk->next;
+    }
+    offset = seek;
+    seek = (size_t)undo->reader.offset;
+  }
+
+  if (chunk) {
+    totread = 0;
+
+    do {
+      /* First check if it's on the end if current chunk. */
+      if (seek - offset == chunk->size) {
+        offset += chunk->size;
+        chunk = chunk->next;
+      }
+
+      /* Debug, should never happen. */
+      if (chunk == NULL) {
+        printf("illegal read, chunk zero\n");
+        return 0;
+      }
+
+      chunkoffset = seek - offset;
+      readsize = size - totread;
+
+      /* Data can be spread over multiple chunks, so clamp size
+       * to within this chunk, and then it will read further in
+       * the next chunk. */
+      if (chunkoffset + readsize > chunk->size) {
+        readsize = chunk->size - chunkoffset;
+      }
+
+      memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
+      totread += readsize;
+      undo->reader.offset += (off64_t)readsize;
+      seek += readsize;
+
+      /* `is_identical` of current chunk represents whether it changed compared to previous undo
+       * step. this is fine in redo case, but not in undo case, where we need an extra flag
+       * defined when saving the next (future) step after the one we want to restore, as we are
+       * supposed to 'come from' that future undo step, and not the one before current one. */
+      undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical :
+                                                                      chunk->is_identical_future;
+    } while (totread < size);
+
+    return (ssize_t)totread;
+  }
+
+  return 0;
+}
+
+static void undo_close(FileReader *reader)
+{
+  MEM_freeN(reader);
+}
+
+FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction)
+{
+  UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__);
+
+  undo->memfile = memfile;
+  undo->undo_direction = undo_direction;
+
+  undo->reader.read = undo_read;
+  undo->reader.seek = NULL;
+  undo->reader.close = undo_close;
+
+  return (FileReader *)undo;
+}
diff --git a/source/blender/blenloader/intern/versioning_250.c b/source/blender/blenloader/intern/versioning_250.c
index e56c1995363..436645c2241 100644
--- a/source/blender/blenloader/intern/versioning_250.c
+++ b/source/blender/blenloader/intern/versioning_250.c
@@ -23,8 +23,7 @@
 #else
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
-#  include <io.h>   /* for open close read */
-#  include <zlib.h> /* odd include order-issue */
+#  include <io.h> /* for open close read */
 #endif
 
 /* allow readfile to use deprecated functionality */
diff --git a/source/blender/blenloader/intern/versioning_legacy.c b/source/blender/blenloader/intern/versioning_legacy.c
index 81371e1c1ed..6ba27b6ee9e 100644
--- a/source/blender/blenloader/intern/versioning_legacy.c
+++ b/source/blender/blenloader/intern/versioning_legacy.c
@@ -28,8 +28,7 @@
 #else
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
-#  include <io.h>   /* for open close read */
-#  include <zlib.h> /* odd include order-issue */
+#  include <io.h> /* for open close read */
 #endif
 
 /* allow readfile to use deprecated functionality */
diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 99246603e9a..4de8e35f402 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -78,12 +78,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <zlib.h>
 
 #ifdef WIN32
 #  include "BLI_winstuff.h"
 #  include "winsock2.h"
 #  include <io.h>
-#  include <zlib.h> /* odd include order-issue */
 #else
 #  include <unistd.h> /* FreeBSD, for write() and close(). */
 #endif
diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c
index b53ad0ee927..c3789abf2dd 100644
--- a/source/blender/windowmanager/intern/wm_files.c
+++ b/source/blender/windowmanager/intern/wm_files.c
@@ -28,11 +28,10 @@
  * winsock stuff.
  */
 #include <errno.h>
+#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
 #include <stddef.h>
 #include <string.h>
 
-#include "zlib.h" /* wm_read_exotic() */
-
 #ifdef WIN32
 /* Need to include windows.h so _WIN32_IE is defined. */
 #  include <windows.h>
@@ -51,6 +50,7 @@
 
 #include "BLI_blenlib.h"
 #include "BLI_fileops_types.h"
+#include "BLI_filereader.h"
 #include "BLI_linklist.h"
 #include "BLI_math.h"
 #include "BLI_system.h"
@@ -481,53 +481,59 @@ static void wm_init_userdef(Main *bmain)
 /* intended to check for non-blender formats but for now it only reads blends */
 static int wm_read_exotic(const char *name)
 {
-  int len;
-  gzFile gzfile;
+  /* make sure we're not trying to read a directory.... */
+
+  int namelen = strlen(name);
+  if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) {
+    return BKE_READ_EXOTIC_FAIL_PATH;
+  }
+
+  /* open the file. */
+  const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0);
+  if (filedes == -1) {
+    return BKE_READ_EXOTIC_FAIL_OPEN;
+  }
+
+  FileReader *rawfile = BLI_filereader_new_file(filedes);
+  if (rawfile == NULL) {
+    return BKE_READ_EXOTIC_FAIL_OPEN;
+  }
+
+  /* read the header (7 bytes are enough to identify all known types). */
   char header[7];
-  int retval;
+  if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
+    rawfile->close(rawfile);
+    return BKE_READ_EXOTIC_FAIL_FORMAT;
+  }
+  rawfile->seek(rawfile, 0, SEEK_SET);
 
-  /* make sure we're not trying to read a directory.... */
+  /* check for uncompressed .blend */
+  if (STREQLEN(header, "BLENDER", 7)) {
+    rawfile->close(rawfile);
+    return BKE_READ_EXOTIC_OK_BLEND;
+  }
 
-  len = strlen(name);
-  if (len > 0 && ELEM(name[len - 1], '/', '\\')) {
-    retval = BKE_READ_EXOTIC_FAIL_PATH;
+  /* check for compressed .blend */
+  FileReader *compressed_file = NULL;
+  if (BLI_file_magic_is_gzip(header)) {
+    compressed_file = BLI_filereader_new_gzip(rawfile);
   }
-  else {
-    gzfile = BLI_gzopen(name, "rb");
-    if (gzfile == NULL) {
-      retval = BKE_READ_EXOTIC_FAIL_OPEN;
-    }
-    else {
-      len = gzread(gzfile, header, sizeof(header));
-      gzclose(gzfile);
-      if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
-        retval = BKE_READ_EXOTIC_OK_BLEND;
-      }
-      else {
-        /* We may want to support loading other file formats
-         * from their header bytes or file extension.
-         * This used to be supported in the code below and may be added
-         * back at some point. */
-#if 0
-        WM_cursor_wait(true);
 
-        if (is_foo_format(name)) {
-          read_foo(name);
-          retval = BKE_READ_EXOTIC_OK_OTHER;
-        }
-        else
-#endif
-        {
-          retval = BKE_READ_EXOTIC_FAIL_FORMAT;
-        }
-#if 0
-        WM_cursor_wait(false);
-#endif
-      }
+  /* If a compression signature matches, try decompressing the start and check if it's a .blend */
+  if (compressed_file != NULL) {
+    size_t len = compressed_file->read(compressed_file, header, sizeof(header));
+    compressed_file->close(compressed_file);
+    if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
+      return BKE_READ_EXOTIC_OK_BLEND;
     }
   }
+  else {
+    rawfile->close(rawfile);
+  }
+
+  /* Add check for future file formats here. */
 
-  return retval;
+  return BKE_READ_EXOTIC_FAIL_FORMAT;
 }
 
 /** \} */
-- 
cgit v1.2.3


From 2ea66af742bca4b427f88de13254414730a33776 Mon Sep 17 00:00:00 2001
From: Lukas Stockner <lukas.stockner@freenet.de>
Date: Sat, 21 Aug 2021 03:15:31 +0200
Subject: Add support for Zstandard compression for .blend files

Compressing blendfiles can help save a lot of disk space, but the slowdown
while loading and saving is a major annoyance.
Currently Blender uses Zlib (aka gzip aka Deflate) for compression, but there
are now several more modern algorithms that outperform it in every way.

In this patch, I decided for Zstandard aka Zstd for several reasons:
- It is widely supported, both in other programs and libraries as well as in
  general-purpose compression utilities on Unix
- It is extremely flexible - spanning several orders of magnitude of
  compression speeds depending on the level setting.
- It is pretty much on the Pareto frontier for all of its configurations
  (meaning that no other algorithm is both faster and more efficient).

One downside of course is that older versions of Blender will not be able to
read these files, but one can always just re-save them without compression or
decompress the file manually with an external tool.

The implementation here saves additional metadata into the compressed file in
order to allow for efficient seeking when loading. This is standard-compliant
and will be ignored by other tools that support Zstd.
If the metadata is not present (e.g. because you manually compressed a .blend
file with another tool), Blender will fall back to sequential reading.

Saving is multithreaded to improve performance. Loading is currently not
multithreaded since it's not easy to predict the access patterns of the
loading code when seeking is supported.
In the future, we might want to look into making this more predictable or
disabling seeking for the main .blend file, which would then allow for
multiple background threads that decompress data ahead of time.

The compression level was chosen to get sizes comparable to previous versions
at much higher speeds. In the future, this could be exposed as an option.

Reviewed By: campbellbarton, brecht, mont29

Differential Revision: https://developer.blender.org/D5799
---
 build_files/cmake/Modules/FindZstd.cmake        |  66 +++++
 build_files/cmake/platform/platform_apple.cmake |   3 +
 build_files/cmake/platform/platform_unix.cmake  |   1 +
 build_files/cmake/platform/platform_win32.cmake |   3 +
 source/blender/blenlib/BLI_fileops.h            |   2 +
 source/blender/blenlib/BLI_filereader.h         |   4 +-
 source/blender/blenlib/CMakeLists.txt           |   3 +
 source/blender/blenlib/intern/fileops.c         |  27 ++
 source/blender/blenlib/intern/filereader_zstd.c | 335 ++++++++++++++++++++++++
 source/blender/blenloader/CMakeLists.txt        |   2 +-
 source/blender/blenloader/intern/readfile.c     |   9 +
 source/blender/blenloader/intern/writefile.c    | 297 ++++++++++++++++-----
 source/blender/windowmanager/CMakeLists.txt     |   4 -
 source/blender/windowmanager/intern/wm_files.c  |   5 +
 14 files changed, 689 insertions(+), 72 deletions(-)
 create mode 100644 build_files/cmake/Modules/FindZstd.cmake
 create mode 100644 source/blender/blenlib/intern/filereader_zstd.c

diff --git a/build_files/cmake/Modules/FindZstd.cmake b/build_files/cmake/Modules/FindZstd.cmake
new file mode 100644
index 00000000000..84606d01f44
--- /dev/null
+++ b/build_files/cmake/Modules/FindZstd.cmake
@@ -0,0 +1,66 @@
+# - Find Zstd library
+# Find the native Zstd includes and library
+# This module defines
+#  ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when
+#                     ZSTD_INCLUDE_DIR is found.
+#  ZSTD_LIBRARIES, libraries to link against to use Zstd.
+#  ZSTD_ROOT_DIR, The base directory to search for Zstd.
+#                 This can also be an environment variable.
+#  ZSTD_FOUND, If false, do not try to use Zstd.
+#
+# also defined, but not for general use are
+#  ZSTD_LIBRARY, where to find the Zstd library.
+
+#=============================================================================
+# Copyright 2019 Blender Foundation.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+
+# If ZSTD_ROOT_DIR was defined in the environment, use it.
+IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "")
+  SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR})
+ENDIF()
+
+SET(_zstd_SEARCH_DIRS
+  ${ZSTD_ROOT_DIR}
+)
+
+FIND_PATH(ZSTD_INCLUDE_DIR
+  NAMES
+    zstd.h
+  HINTS
+    ${_zstd_SEARCH_DIRS}
+  PATH_SUFFIXES
+    include
+)
+
+FIND_LIBRARY(ZSTD_LIBRARY
+  NAMES
+    zstd
+  HINTS
+    ${_zstd_SEARCH_DIRS}
+  PATH_SUFFIXES
+    lib64 lib
+  )
+
+# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG
+    ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+IF(ZSTD_FOUND)
+  SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+  SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+ENDIF()
+
+MARK_AS_ADVANCED(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY
+)
diff --git a/build_files/cmake/platform/platform_apple.cmake b/build_files/cmake/platform/platform_apple.cmake
index dceafb236de..2bfdc84ec2a 100644
--- a/build_files/cmake/platform/platform_apple.cmake
+++ b/build_files/cmake/platform/platform_apple.cmake
@@ -441,6 +441,9 @@ if(WITH_HARU)
   endif()
 endif()
 
+set(ZSTD_ROOT_DIR ${LIBDIR}/zstd)
+find_package(Zstd REQUIRED)
+
 if(EXISTS ${LIBDIR})
   without_system_libs_end()
 endif()
diff --git a/build_files/cmake/platform/platform_unix.cmake b/build_files/cmake/platform/platform_unix.cmake
index 7f62399ac4f..fc0c37e4c8b 100644
--- a/build_files/cmake/platform/platform_unix.cmake
+++ b/build_files/cmake/platform/platform_unix.cmake
@@ -99,6 +99,7 @@ endif()
 find_package_wrapper(JPEG REQUIRED)
 find_package_wrapper(PNG REQUIRED)
 find_package_wrapper(ZLIB REQUIRED)
+find_package_wrapper(Zstd REQUIRED)
 find_package_wrapper(Freetype REQUIRED)
 
 if(WITH_PYTHON)
diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake
index 3773aaaffed..d44ef691d1b 100644
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@@ -873,3 +873,6 @@ if(WITH_HARU)
     set(WITH_HARU OFF)
   endif()
 endif()
+
+set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include)
+set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib)
diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 12fa73279c8..377b7bc3bc2 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -168,6 +168,8 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g
     ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 bool BLI_file_magic_is_gzip(const char header[4]);
 
+bool BLI_file_magic_is_zstd(const char header[4]);
+
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
 size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 
diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h
index dbcaa7cb1b6..8d1fa3d1596 100644
--- a/source/blender/blenlib/BLI_filereader.h
+++ b/source/blender/blenlib/BLI_filereader.h
@@ -61,7 +61,7 @@ typedef struct FileReader {
  *
  * If a FileReader is created, it has to be cleaned up and freed by calling
  * its close() function unless another FileReader has taken ownership - for example,
- * Gzip takes over the base FileReader and will clean it up when their clean() is called.
+ * Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called.
  */
 
 /* Create FileReader from raw file descriptor. */
@@ -71,6 +71,8 @@ FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
 /* Create FileReader from a region of memory. */
 FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
     ATTR_NONNULL();
+/* Create FileReader from applying Zstd decompression on an underlying file. */
+FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 /* Create FileReader from applying Gzip decompression on an underlying file. */
 FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index d2ba9e74c90..f98d15ad08b 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -31,6 +31,7 @@ set(INC
 
 set(INC_SYS
   ${ZLIB_INCLUDE_DIRS}
+  ${ZSTD_INCLUDE_DIRS}
   ${FREETYPE_INCLUDE_DIRS}
   ${GMP_INCLUDE_DIRS}
 )
@@ -78,6 +79,7 @@ set(SRC
   intern/filereader_file.c
   intern/filereader_gzip.c
   intern/filereader_memory.c
+  intern/filereader_zstd.c
   intern/fnmatch.c
   intern/freetypefont.c
   intern/gsqueue.c
@@ -327,6 +329,7 @@ set(LIB
 
   ${FREETYPE_LIBRARY}
   ${ZLIB_LIBRARIES}
+  ${ZSTD_LIBRARIES}
 )
 
 if(WITH_MEM_VALGRIND)
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 6fc2222241b..31825c69737 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -262,6 +262,33 @@ bool BLI_file_magic_is_gzip(const char header[4])
   return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
 }
 
+bool BLI_file_magic_is_zstd(const char header[4])
+{
+  /* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame.
+   * Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5*
+   * for skippable frames, with the * being anything from 0 to F.
+   *
+   * To check whether a file is Zstd-compressed, we just check whether the first frame matches
+   * either. Seeking through the file until a Zstd frame is found would make things more
+   * complicated and the probability of a false positive is rather low anyways.
+   *
+   * Note that LZ4 uses a compatible format, so even though its compressed frames have a
+   * different magic number, a valid LZ4 file might also start with a skippable frame matching
+   * the second check here.
+   *
+   * For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+   */
+
+  uint32_t magic = *((uint32_t *)header);
+  if (magic == 0xFD2FB528) {
+    return true;
+  }
+  if ((magic >> 4) == 0x184D2A5) {
+    return true;
+  }
+  return false;
+}
+
 /**
  * Returns true if the file with the specified name can be written.
  * This implementation uses access(2), which makes the check according
diff --git a/source/blender/blenlib/intern/filereader_zstd.c b/source/blender/blenlib/intern/filereader_zstd.c
new file mode 100644
index 00000000000..785a40cd1a1
--- /dev/null
+++ b/source/blender/blenlib/intern/filereader_zstd.c
@@ -0,0 +1,335 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include <string.h>
+#include <zstd.h>
+
+#include "BLI_blenlib.h"
+#include "BLI_endian_switch.h"
+#include "BLI_filereader.h"
+#include "BLI_math_base.h"
+
+#include "MEM_guardedalloc.h"
+
+typedef struct {
+  FileReader reader;
+
+  FileReader *base;
+
+  ZSTD_DCtx *ctx;
+  ZSTD_inBuffer in_buf;
+  size_t in_buf_max_size;
+
+  struct {
+    int num_frames;
+    size_t *compressed_ofs;
+    size_t *uncompressed_ofs;
+
+    char *cached_content;
+    int cached_frame;
+  } seek;
+} ZstdReader;
+
+static bool zstd_read_u32(FileReader *base, uint32_t *val)
+{
+  if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
+    return false;
+  }
+#ifdef __BIG_ENDIAN__
+  BLI_endian_switch_uint32(val);
+#endif
+  return true;
+}
+
+static bool zstd_read_seek_table(ZstdReader *zstd)
+{
+  FileReader *base = zstd->base;
+
+  /* The seek table frame is at the end of the file, so seek there
+   * and verify that there is enough data. */
+  if (base->seek(base, -4, SEEK_END) < 13) {
+    return false;
+  }
+  uint32_t magic;
+  if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
+    return false;
+  }
+
+  uint8_t flags;
+  if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
+    return false;
+  }
+  /* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */
+  bool has_checksums = (flags & 0x80);
+  if (flags & 0x60) {
+    return false;
+  }
+
+  uint32_t num_frames;
+  if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &num_frames)) {
+    return false;
+  }
+
+  /* Each frame has either 2 or 3 uint32_t, and after that we have
+   * num_frames, flags and magic for another 9 bytes. */
+  uint32_t expected_frame_length = num_frames * (has_checksums ? 12 : 8) + 9;
+  /* The frame starts with another magic number and its length, but these
+   * two fields are not included when counting length. */
+  off64_t frame_start_ofs = 8 + expected_frame_length;
+  /* Sanity check: Before the start of the seek table frame,
+   * there must be num_frames frames, each of which at least 8 bytes long. */
+  off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END);
+  if (seek_frame_start < num_frames * 8) {
+    return false;
+  }
+
+  if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) {
+    return false;
+  }
+
+  uint32_t frame_length;
+  if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) {
+    return false;
+  }
+
+  zstd->seek.num_frames = num_frames;
+  zstd->seek.compressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
+  zstd->seek.uncompressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
+
+  size_t compressed_ofs = 0;
+  size_t uncompressed_ofs = 0;
+  for (int i = 0; i < num_frames; i++) {
+    uint32_t compressed_size, uncompressed_size;
+    if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) {
+      break;
+    }
+    if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) {
+      break;
+    }
+    zstd->seek.compressed_ofs[i] = compressed_ofs;
+    zstd->seek.uncompressed_ofs[i] = uncompressed_ofs;
+    compressed_ofs += compressed_size;
+    uncompressed_ofs += uncompressed_size;
+  }
+  zstd->seek.compressed_ofs[num_frames] = compressed_ofs;
+  zstd->seek.uncompressed_ofs[num_frames] = uncompressed_ofs;
+
+  /* Seek to the end of the previous frame for the following BHead frame detection. */
+  if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) {
+    MEM_freeN(zstd->seek.compressed_ofs);
+    MEM_freeN(zstd->seek.uncompressed_ofs);
+    memset(&zstd->seek, 0, sizeof(zstd->seek));
+    return false;
+  }
+
+  zstd->seek.cached_frame = -1;
+
+  return true;
+}
+
+/* Find out which frame contains the given position in the uncompressed stream.
+ * Basically just bisection. */
+static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
+{
+  int low = 0, high = zstd->seek.num_frames;
+
+  if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
+    return -1;
+  }
+
+  while (low + 1 < high) {
+    int mid = low + ((high - low) >> 1);
+    if (zstd->seek.uncompressed_ofs[mid] <= pos) {
+      low = mid;
+    }
+    else {
+      high = mid;
+    }
+  }
+
+  return low;
+}
+
+/* Ensure that the currently loaded frame is the correct one. */
+static const char *zstd_ensure_cache(ZstdReader *zstd, int frame)
+{
+  if (zstd->seek.cached_frame == frame) {
+    /* Cached frame matches, so just return it. */
+    return zstd->seek.cached_content;
+  }
+
+  /* Cached frame doesn't match, so discard it and cache the wanted one onstead. */
+  MEM_SAFE_FREE(zstd->seek.cached_content);
+
+  size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame];
+  size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] -
+                             zstd->seek.uncompressed_ofs[frame];
+
+  char *uncompressed_data = MEM_mallocN(uncompressed_size, __func__);
+  char *compressed_data = MEM_mallocN(compressed_size, __func__);
+  if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 ||
+      zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size) {
+    MEM_freeN(compressed_data);
+    MEM_freeN(uncompressed_data);
+    return NULL;
+  }
+
+  size_t res = ZSTD_decompressDCtx(
+      zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size);
+  MEM_freeN(compressed_data);
+  if (ZSTD_isError(res) || res < uncompressed_size) {
+    MEM_freeN(uncompressed_data);
+    return NULL;
+  }
+
+  zstd->seek.cached_frame = frame;
+  zstd->seek.cached_content = uncompressed_data;
+  return uncompressed_data;
+}
+
+static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+
+  size_t end_offset = zstd->reader.offset + size, read_len = 0;
+  while (zstd->reader.offset < end_offset) {
+    int frame = zstd_frame_from_pos(zstd, zstd->reader.offset);
+    if (frame < 0) {
+      /* EOF is reached, so return as much as we can. */
+      break;
+    }
+
+    const char *framedata = zstd_ensure_cache(zstd, frame);
+    if (framedata == NULL) {
+      /* Error while reading the frame, so return as much as we can. */
+      break;
+    }
+
+    size_t frame_end_offset = min_zz(zstd->seek.uncompressed_ofs[frame + 1], end_offset);
+    size_t frame_read_len = frame_end_offset - zstd->reader.offset;
+
+    size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame];
+    memcpy((char *)buffer + read_len, framedata + offset_in_frame, frame_read_len);
+    read_len += frame_read_len;
+    zstd->reader.offset = frame_end_offset;
+  }
+
+  return read_len;
+}
+
+static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+  off64_t new_pos;
+  if (whence == SEEK_SET) {
+    new_pos = offset;
+  }
+  else if (whence == SEEK_END) {
+    new_pos = zstd->seek.uncompressed_ofs[zstd->seek.num_frames] + offset;
+  }
+  else {
+    new_pos = zstd->reader.offset + offset;
+  }
+
+  if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
+    return -1;
+  }
+  zstd->reader.offset = new_pos;
+  return zstd->reader.offset;
+}
+
+static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+  ZSTD_outBuffer output = {buffer, size, 0};
+
+  while (output.pos < output.size) {
+    if (zstd->in_buf.pos == zstd->in_buf.size) {
+      /* Ran out of buffered input data, read some more. */
+      zstd->in_buf.pos = 0;
+      ssize_t readsize = zstd->base->read(
+          zstd->base, (char *)zstd->in_buf.src, zstd->in_buf_max_size);
+
+      if (readsize > 0) {
+        /* We got some data, so mark the buffer as refilled. */
+        zstd->in_buf.size = readsize;
+      }
+      else {
+        /* The underlying file is EOF, so return as much as we can. */
+        break;
+      }
+    }
+
+    if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) {
+      break;
+    }
+  }
+
+  zstd->reader.offset += output.pos;
+  return output.pos;
+}
+
+static void zstd_close(FileReader *reader)
+{
+  ZstdReader *zstd = (ZstdReader *)reader;
+
+  ZSTD_freeDCtx(zstd->ctx);
+  if (zstd->reader.seek) {
+    MEM_freeN(zstd->seek.uncompressed_ofs);
+    MEM_freeN(zstd->seek.compressed_ofs);
+    MEM_freeN(zstd->seek.cached_content);
+  }
+  else {
+    MEM_freeN((void *)zstd->in_buf.src);
+  }
+
+  zstd->base->close(zstd->base);
+  MEM_freeN(zstd);
+}
+
+FileReader *BLI_filereader_new_zstd(FileReader *base)
+{
+  ZstdReader *zstd = MEM_callocN(sizeof(ZstdReader), __func__);
+
+  zstd->ctx = ZSTD_createDCtx();
+  zstd->base = base;
+
+  if (zstd_read_seek_table(zstd)) {
+    zstd->reader.read = zstd_read_seekable;
+    zstd->reader.seek = zstd_seek;
+  }
+  else {
+    zstd->reader.read = zstd_read;
+    zstd->reader.seek = NULL;
+
+    zstd->in_buf_max_size = ZSTD_DStreamInSize();
+    zstd->in_buf.src = MEM_mallocN(zstd->in_buf_max_size, "zstd in buf");
+    zstd->in_buf.size = zstd->in_buf_max_size;
+    /* This signals that the buffer has run out,
+     * which will make the read function refill it on the first call. */
+    zstd->in_buf.pos = zstd->in_buf_max_size;
+  }
+  zstd->reader.close = zstd_close;
+
+  return (FileReader *)zstd;
+}
diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt
index f5baf0dcb83..89631588ed0 100644
--- a/source/blender/blenloader/CMakeLists.txt
+++ b/source/blender/blenloader/CMakeLists.txt
@@ -42,7 +42,7 @@ set(INC
 )
 
 set(INC_SYS
-  ${ZLIB_INCLUDE_DIRS}
+  ${ZSTD_INCLUDE_DIRS}
 )
 
 set(SRC
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 82fbb2083df..49c3497f996 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -1233,6 +1233,12 @@ static FileData *blo_filedata_from_file_descriptor(const char *filepath,
       rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
     }
   }
+  else if (BLI_file_magic_is_zstd(header)) {
+    file = BLI_filereader_new_zstd(rawfile);
+    if (file != NULL) {
+      rawfile = NULL; /* The Zstd FileReader takes ownership of `rawfile`. */
+    }
+  }
 
   /* Clean up `rawfile` if it wasn't taken over. */
   if (rawfile != NULL) {
@@ -1309,6 +1315,9 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
   if (BLI_file_magic_is_gzip(mem)) {
     file = BLI_filereader_new_gzip(mem_file);
   }
+  else if (BLI_file_magic_is_zstd(mem)) {
+    file = BLI_filereader_new_zstd(mem_file);
+  }
 
   if (file == NULL) {
     /* Compression initialization failed. */
diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 4de8e35f402..6f43fbf1fa0 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -101,7 +101,12 @@
 #include "BLI_bitmap.h"
 #include "BLI_blenlib.h"
 #include "BLI_endian_defines.h"
+#include "BLI_endian_switch.h"
+#include "BLI_link_utils.h"
+#include "BLI_linklist.h"
+#include "BLI_math_base.h"
 #include "BLI_mempool.h"
+#include "BLI_threads.h"
 #include "MEM_guardedalloc.h" /* MEM_freeN */
 
 #include "BKE_blender_version.h"
@@ -129,14 +134,21 @@
 
 #include <errno.h>
 
+#include <zstd.h>
+
 /* Make preferences read-only. */
 #define U (*((const UserDef *)&U))
 
 /* ********* my write, buffered writing with minimum size chunks ************ */
 
 /* Use optimal allocation since blocks of this size are kept in memory for undo. */
-#define MYWRITE_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
-#define MYWRITE_MAX_CHUNK (MEM_SIZE_OPTIMAL(1 << 15))   /* ~32kb */
+#define MEM_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
+#define MEM_CHUNK_SIZE (MEM_SIZE_OPTIMAL(1 << 15))  /* ~32kb */
+
+#define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */
+#define ZSTD_CHUNK_SIZE (1 << 20)  /* 1mb */
+
+#define ZSTD_COMPRESSION_LEVEL 3
 
 /** Use if we want to store how many bytes have been written to the file. */
 // #define USE_WRITE_DATA_LEN
@@ -147,9 +159,16 @@
 
 typedef enum {
   WW_WRAP_NONE = 1,
-  WW_WRAP_ZLIB,
+  WW_WRAP_ZSTD,
 } eWriteWrapType;
 
+typedef struct ZstdFrame {
+  struct ZstdFrame *next, *prev;
+
+  uint32_t compressed_size;
+  uint32_t uncompressed_size;
+} ZstdFrame;
+
 typedef struct WriteWrap WriteWrap;
 struct WriteWrap {
   /* callbacks */
@@ -161,15 +180,23 @@ struct WriteWrap {
   bool use_buf;
 
   /* internal */
-  union {
-    int file_handle;
-    gzFile gz_handle;
-  } _user_data;
+  int file_handle;
+  struct {
+    ListBase threadpool;
+    ListBase tasks;
+    ThreadMutex mutex;
+    ThreadCondition condition;
+    int next_frame;
+    int num_frames;
+
+    int level;
+    ListBase frames;
+
+    bool write_error;
+  } zstd;
 };
 
 /* none */
-#define FILE_HANDLE(ww) (ww)->_user_data.file_handle
-
 static bool ww_open_none(WriteWrap *ww, const char *filepath)
 {
   int file;
@@ -177,7 +204,7 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
   file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666);
 
   if (file != -1) {
-    FILE_HANDLE(ww) = file;
+    ww->file_handle = file;
     return true;
   }
 
@@ -185,39 +212,170 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
 }
 static bool ww_close_none(WriteWrap *ww)
 {
-  return (close(FILE_HANDLE(ww)) != -1);
+  return (close(ww->file_handle) != -1);
 }
 static size_t ww_write_none(WriteWrap *ww, const char *buf, size_t buf_len)
 {
-  return write(FILE_HANDLE(ww), buf, buf_len);
+  return write(ww->file_handle, buf, buf_len);
 }
-#undef FILE_HANDLE
 
-/* zlib */
-#define FILE_HANDLE(ww) (ww)->_user_data.gz_handle
+/* zstd */
+
+typedef struct {
+  struct ZstdWriteBlockTask *next, *prev;
+  void *data;
+  size_t size;
+  int frame_number;
+  WriteWrap *ww;
+} ZstdWriteBlockTask;
 
-static bool ww_open_zlib(WriteWrap *ww, const char *filepath)
+static void *zstd_write_task(void *userdata)
 {
-  gzFile file;
+  ZstdWriteBlockTask *task = userdata;
+  WriteWrap *ww = task->ww;
 
-  file = BLI_gzopen(filepath, "wb1");
+  size_t out_buf_len = ZSTD_compressBound(task->size);
+  void *out_buf = MEM_mallocN(out_buf_len, "Zstd out buffer");
+  size_t out_size = ZSTD_compress(
+      out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL);
 
-  if (file != Z_NULL) {
-    FILE_HANDLE(ww) = file;
-    return true;
+  MEM_freeN(task->data);
+
+  BLI_mutex_lock(&ww->zstd.mutex);
+
+  while (ww->zstd.next_frame != task->frame_number) {
+    BLI_condition_wait(&ww->zstd.condition, &ww->zstd.mutex);
   }
 
-  return false;
+  if (ZSTD_isError(out_size)) {
+    ww->zstd.write_error = true;
+  }
+  else {
+    if (ww_write_none(ww, out_buf, out_size) == out_size) {
+      ZstdFrame *frameinfo = MEM_mallocN(sizeof(ZstdFrame), "zstd frameinfo");
+      frameinfo->uncompressed_size = task->size;
+      frameinfo->compressed_size = out_size;
+      BLI_addtail(&ww->zstd.frames, frameinfo);
+    }
+    else {
+      ww->zstd.write_error = true;
+    }
+  }
+
+  ww->zstd.next_frame++;
+
+  BLI_mutex_unlock(&ww->zstd.mutex);
+  BLI_condition_notify_all(&ww->zstd.condition);
+
+  MEM_freeN(out_buf);
+  return NULL;
+}
+
+static bool ww_open_zstd(WriteWrap *ww, const char *filepath)
+{
+  if (!ww_open_none(ww, filepath)) {
+    return false;
+  }
+
+  /* Leave one thread open for the main writing logic, unless we only have one HW thread. */
+  int num_threads = max_ii(1, BLI_system_thread_count() - 1);
+  BLI_threadpool_init(&ww->zstd.threadpool, zstd_write_task, num_threads);
+  BLI_mutex_init(&ww->zstd.mutex);
+  BLI_condition_init(&ww->zstd.condition);
+
+  return true;
 }
-static bool ww_close_zlib(WriteWrap *ww)
+
+static void zstd_write_u32_le(WriteWrap *ww, uint32_t val)
+{
+#ifdef __BIG_ENDIAN__
+  BLI_endian_switch_uint32(&val);
+#endif
+  ww_write_none(ww, (char *)&val, sizeof(uint32_t));
+}
+
+/* In order to implement efficient seeking when reading the .blend, we append
+ * a skippable frame that encodes information about the other frames present
+ * in the file.
+ * The format here follows the upstream spec for seekable files:
+ * https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md
+ * If this information is not present in a file (e.g. if it was compressed
+ * with external tools), it can still be opened in Blender, but seeking will
+ * not be supported, so more memory might be needed. */
+static void zstd_write_seekable_frames(WriteWrap *ww)
+{
+  /* Write seek table header (magic number and frame size). */
+  zstd_write_u32_le(ww, 0x184D2A5E);
+
+  /* The actual frame number might not match ww->zstd.num_frames if there was a write error. */
+  const uint32_t num_frames = BLI_listbase_count(&ww->zstd.frames);
+  /* Each frame consists of two u32, so 8 bytes each.
+   * After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */
+  const uint32_t frame_size = num_frames * 8 + 9;
+  zstd_write_u32_le(ww, frame_size);
+
+  /* Write seek table entries. */
+  LISTBASE_FOREACH (ZstdFrame *, frame, &ww->zstd.frames) {
+    zstd_write_u32_le(ww, frame->compressed_size);
+    zstd_write_u32_le(ww, frame->uncompressed_size);
+  }
+
+  /* Write seek table footer (number of frames, option flags and second magic number). */
+  zstd_write_u32_le(ww, num_frames);
+  const char flags = 0; /* We don't store checksums for each frame. */
+  ww_write_none(ww, &flags, 1);
+  zstd_write_u32_le(ww, 0x8F92EAB1);
+}
+
+static bool ww_close_zstd(WriteWrap *ww)
 {
-  return (gzclose(FILE_HANDLE(ww)) == Z_OK);
+  BLI_threadpool_end(&ww->zstd.threadpool);
+  BLI_freelistN(&ww->zstd.tasks);
+
+  BLI_mutex_end(&ww->zstd.mutex);
+  BLI_condition_end(&ww->zstd.condition);
+
+  zstd_write_seekable_frames(ww);
+  BLI_freelistN(&ww->zstd.frames);
+
+  return ww_close_none(ww) && !ww->zstd.write_error;
 }
-static size_t ww_write_zlib(WriteWrap *ww, const char *buf, size_t buf_len)
+
+static size_t ww_write_zstd(WriteWrap *ww, const char *buf, size_t buf_len)
 {
-  return gzwrite(FILE_HANDLE(ww), buf, buf_len);
+  if (ww->zstd.write_error) {
+    return 0;
+  }
+
+  ZstdWriteBlockTask *task = MEM_mallocN(sizeof(ZstdWriteBlockTask), __func__);
+  task->data = MEM_mallocN(buf_len, __func__);
+  memcpy(task->data, buf, buf_len);
+  task->size = buf_len;
+  task->frame_number = ww->zstd.num_frames++;
+  task->ww = ww;
+
+  BLI_mutex_lock(&ww->zstd.mutex);
+  BLI_addtail(&ww->zstd.tasks, task);
+
+  /* If there's a free worker thread, just push the block into that thread.
+   * Otherwise, we wait for the earliest thread to finish.
+   * We look up the earliest thread while holding the mutex, but release it
+   * before joining the thread to prevent a deadlock. */
+  ZstdWriteBlockTask *first_task = ww->zstd.tasks.first;
+  BLI_mutex_unlock(&ww->zstd.mutex);
+  if (!BLI_available_threads(&ww->zstd.threadpool)) {
+    BLI_threadpool_remove(&ww->zstd.threadpool, first_task);
+
+    /* If the task list was empty before we pushed our task, there should
+     * always be a free thread. */
+    BLI_assert(first_task != task);
+    BLI_remlink(&ww->zstd.tasks, first_task);
+    MEM_freeN(first_task);
+  }
+  BLI_threadpool_insert(&ww->zstd.threadpool, task);
+
+  return buf_len;
 }
-#undef FILE_HANDLE
 
 /* --- end compression types --- */
 
@@ -226,11 +384,11 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
   memset(r_ww, 0, sizeof(*r_ww));
 
   switch (ww_type) {
-    case WW_WRAP_ZLIB: {
-      r_ww->open = ww_open_zlib;
-      r_ww->close = ww_close_zlib;
-      r_ww->write = ww_write_zlib;
-      r_ww->use_buf = false;
+    case WW_WRAP_ZSTD: {
+      r_ww->open = ww_open_zstd;
+      r_ww->close = ww_close_zstd;
+      r_ww->write = ww_write_zstd;
+      r_ww->use_buf = true;
       break;
     }
     default: {
@@ -252,10 +410,17 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
 typedef struct {
   const struct SDNA *sdna;
 
-  /** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */
-  uchar *buf;
-  /** Number of bytes used in #WriteData.buf (flushed when exceeded). */
-  size_t buf_used_len;
+  struct {
+    /** Use for file and memory writing (size stored in max_size). */
+    uchar *buf;
+    /** Number of bytes used in #WriteData.buf (flushed when exceeded). */
+    size_t used_len;
+
+    /** Maximum size of the buffer. */
+    size_t max_size;
+    /** Threshold above which writes get their own chunk. */
+    size_t chunk_size;
+  } buffer;
 
 #ifdef USE_WRITE_DATA_LEN
   /** Total number of bytes written. */
@@ -271,7 +436,7 @@ typedef struct {
   bool use_memfile;
 
   /**
-   * Wrap writing, so we can use zlib or
+   * Wrap writing, so we can use zstd or
    * other compression types later, see: G_FILE_COMPRESS
    * Will be NULL for UNDO.
    */
@@ -291,7 +456,15 @@ static WriteData *writedata_new(WriteWrap *ww)
   wd->ww = ww;
 
   if ((ww == NULL) || (ww->use_buf)) {
-    wd->buf = MEM_mallocN(MYWRITE_BUFFER_SIZE, "wd->buf");
+    if (ww == NULL) {
+      wd->buffer.max_size = MEM_BUFFER_SIZE;
+      wd->buffer.chunk_size = MEM_CHUNK_SIZE;
+    }
+    else {
+      wd->buffer.max_size = ZSTD_BUFFER_SIZE;
+      wd->buffer.chunk_size = ZSTD_CHUNK_SIZE;
+    }
+    wd->buffer.buf = MEM_mallocN(wd->buffer.max_size, "wd->buffer.buf");
   }
 
   return wd;
@@ -325,8 +498,8 @@ static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen)
 
 static void writedata_free(WriteData *wd)
 {
-  if (wd->buf) {
-    MEM_freeN(wd->buf);
+  if (wd->buffer.buf) {
+    MEM_freeN(wd->buffer.buf);
   }
   MEM_freeN(wd);
 }
@@ -343,9 +516,9 @@ static void writedata_free(WriteData *wd)
  */
 static void mywrite_flush(WriteData *wd)
 {
-  if (wd->buf_used_len != 0) {
-    writedata_do_write(wd, wd->buf, wd->buf_used_len);
-    wd->buf_used_len = 0;
+  if (wd->buffer.used_len != 0) {
+    writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+    wd->buffer.used_len = 0;
   }
 }
 
@@ -369,20 +542,20 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
   wd->write_len += len;
 #endif
 
-  if (wd->buf == NULL) {
+  if (wd->buffer.buf == NULL) {
     writedata_do_write(wd, adr, len);
   }
   else {
     /* if we have a single big chunk, write existing data in
      * buffer and write out big chunk in smaller pieces */
-    if (len > MYWRITE_MAX_CHUNK) {
-      if (wd->buf_used_len != 0) {
-        writedata_do_write(wd, wd->buf, wd->buf_used_len);
-        wd->buf_used_len = 0;
+    if (len > wd->buffer.chunk_size) {
+      if (wd->buffer.used_len != 0) {
+        writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+        wd->buffer.used_len = 0;
       }
 
       do {
-        size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK);
+        size_t writelen = MIN2(len, wd->buffer.chunk_size);
         writedata_do_write(wd, adr, writelen);
         adr = (const char *)adr + writelen;
         len -= writelen;
@@ -392,14 +565,14 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
     }
 
     /* if data would overflow buffer, write out the buffer */
-    if (len + wd->buf_used_len > MYWRITE_BUFFER_SIZE - 1) {
-      writedata_do_write(wd, wd->buf, wd->buf_used_len);
-      wd->buf_used_len = 0;
+    if (len + wd->buffer.used_len > wd->buffer.max_size - 1) {
+      writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+      wd->buffer.used_len = 0;
     }
 
     /* append data at end of buffer */
-    memcpy(&wd->buf[wd->buf_used_len], adr, len);
-    wd->buf_used_len += len;
+    memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len);
+    wd->buffer.used_len += len;
   }
 }
 
@@ -430,9 +603,9 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren
  */
 static bool mywrite_end(WriteData *wd)
 {
-  if (wd->buf_used_len != 0) {
-    writedata_do_write(wd, wd->buf, wd->buf_used_len);
-    wd->buf_used_len = 0;
+  if (wd->buffer.used_len != 0) {
+    writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
+    wd->buffer.used_len = 0;
   }
 
   if (wd->use_memfile) {
@@ -1150,7 +1323,6 @@ bool BLO_write_file(Main *mainvar,
                     ReportList *reports)
 {
   char tempname[FILE_MAX + 1];
-  eWriteWrapType ww_type;
   WriteWrap ww;
 
   eBLO_WritePathRemap remap_mode = params->remap_mode;
@@ -1172,14 +1344,7 @@ bool BLO_write_file(Main *mainvar,
   /* open temporary file, so we preserve the original in case we crash */
   BLI_snprintf(tempname, sizeof(tempname), "%s@", filepath);
 
-  if (write_flags & G_FILE_COMPRESS) {
-    ww_type = WW_WRAP_ZLIB;
-  }
-  else {
-    ww_type = WW_WRAP_NONE;
-  }
-
-  ww_handle_init(ww_type, &ww);
+  ww_handle_init((write_flags & G_FILE_COMPRESS) ? WW_WRAP_ZSTD : WW_WRAP_NONE, &ww);
 
   if (ww.open(&ww, tempname) == false) {
     BKE_reportf(
diff --git a/source/blender/windowmanager/CMakeLists.txt b/source/blender/windowmanager/CMakeLists.txt
index b7fbb9bb82b..4d65726fe2b 100644
--- a/source/blender/windowmanager/CMakeLists.txt
+++ b/source/blender/windowmanager/CMakeLists.txt
@@ -48,10 +48,6 @@ set(INC
   ${CMAKE_BINARY_DIR}/source/blender/makesdna/intern
 )
 
-set(INC_SYS
-  ${ZLIB_INCLUDE_DIRS}
-)
-
 set(SRC
   intern/wm.c
   intern/wm_cursors.c
diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c
index c3789abf2dd..f83511e76f0 100644
--- a/source/blender/windowmanager/intern/wm_files.c
+++ b/source/blender/windowmanager/intern/wm_files.c
@@ -516,8 +516,13 @@ static int wm_read_exotic(const char *name)
   /* check for compressed .blend */
   FileReader *compressed_file = NULL;
   if (BLI_file_magic_is_gzip(header)) {
+    /* In earlier versions of Blender (before 3.0), compressed files used Gzip instead of Zstd.
+     * While these files will no longer be written, there still needs to be reading support. */
     compressed_file = BLI_filereader_new_gzip(rawfile);
   }
+  else if (BLI_file_magic_is_zstd(header)) {
+    compressed_file = BLI_filereader_new_zstd(rawfile);
+  }
 
   /* If a compression signature matches, try decompressing the start and check if it's a .blend */
   if (compressed_file != NULL) {
-- 
cgit v1.2.3


From 67c29bc5a273b66e278bd20c18187b425acf1869 Mon Sep 17 00:00:00 2001
From: Lukas Stockner <lukas.stockner@freenet.de>
Date: Sat, 21 Aug 2021 03:26:57 +0200
Subject: Use Zstandard compression for the sequencer cache

Reviewed By: campbellbarton, brecht, mont29

Differential Revision: https://developer.blender.org/D5799
---
 source/blender/blenlib/BLI_fileops.h          |  19 +--
 source/blender/blenlib/intern/fileops.c       | 217 +++++++-------------------
 source/blender/sequencer/intern/image_cache.c |  32 ++--
 3 files changed, 82 insertions(+), 186 deletions(-)

diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 377b7bc3bc2..906a56ce909 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -154,20 +154,15 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL
 bool BLI_file_touch(const char *file) ATTR_NONNULL();
 bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT;
 
-#if 0 /* UNUSED */
-int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-#endif
-char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
-    ATTR_NONNULL();
-size_t BLI_gzip_mem_to_file_at_pos(void *buf,
-                                   size_t len,
-                                   FILE *file,
-                                   size_t gz_stream_offset,
-                                   int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
-size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
-    ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 bool BLI_file_magic_is_gzip(const char header[4]);
 
+size_t BLI_file_zstd_from_mem_at_pos(void *buf,
+                                     size_t len,
+                                     FILE *file,
+                                     size_t file_offset,
+                                     int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
+    ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 bool BLI_file_magic_is_zstd(const char header[4]);
 
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 31825c69737..532a29b8147 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -31,6 +31,7 @@
 #include <errno.h>
 
 #include "zlib.h"
+#include "zstd.h"
 
 #ifdef WIN32
 #  include "BLI_fileops_types.h"
@@ -61,200 +62,90 @@
 #include "BLI_sys_types.h" /* for intptr_t support */
 #include "BLI_utildefines.h"
 
-#if 0 /* UNUSED */
-/* gzip the file in from and write it to "to".
- * return -1 if zlib fails, -2 if the originating file does not exist
- * NOTE: will remove the "from" file
- */
-int BLI_file_gzip(const char *from, const char *to)
+size_t BLI_file_zstd_from_mem_at_pos(
+    void *buf, size_t len, FILE *file, size_t file_offset, int compression_level)
 {
-  char buffer[10240];
-  int file;
-  int readsize = 0;
-  int rval = 0, err;
-  gzFile gzfile;
+  fseek(file, file_offset, SEEK_SET);
 
-  /* level 1 is very close to 3 (the default) in terms of file size,
-   * but about twice as fast, best use for speedy saving - campbell */
-  gzfile = BLI_gzopen(to, "wb1");
-  if (gzfile == NULL) {
-    return -1;
-  }
-  file = BLI_open(from, O_BINARY | O_RDONLY, 0);
-  if (file == -1) {
-    return -2;
-  }
+  ZSTD_CCtx *ctx = ZSTD_createCCtx();
+  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+
+  ZSTD_inBuffer input = {buf, len, 0};
 
-  while (1) {
-    readsize = read(file, buffer, sizeof(buffer));
+  size_t out_len = ZSTD_CStreamOutSize();
+  void *out_buf = MEM_mallocN(out_len, __func__);
+  size_t total_written = 0;
 
-    if (readsize < 0) {
-      rval = -2; /* error happened in reading */
-      fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno));
+  /* Compress block and write it out until the input has been consumed. */
+  while (input.pos < input.size) {
+    ZSTD_outBuffer output = {out_buf, out_len, 0};
+    size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue);
+    if (ZSTD_isError(ret)) {
       break;
     }
-    else if (readsize == 0) {
-      break; /* done reading */
-    }
-
-    if (gzwrite(gzfile, buffer, readsize) <= 0) {
-      rval = -1; /* error happened in writing */
-      fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err));
+    if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
       break;
     }
+    total_written += output.pos;
   }
 
-  gzclose(gzfile);
-  close(file);
-
-  return rval;
-}
-#endif
-
-/* gzip the file in from_file and write it to memory to_mem, at most size bytes.
- * return the unzipped size
- */
-char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
-{
-  gzFile gzfile;
-  int readsize, size, alloc_size = 0;
-  char *mem = NULL;
-  const int chunk_size = 512 * 1024;
-
-  size = 0;
-
-  gzfile = BLI_gzopen(from_file, "rb");
-  for (;;) {
-    if (mem == NULL) {
-      mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem");
-      alloc_size = chunk_size;
-    }
-    else {
-      mem = MEM_reallocN(mem, size + chunk_size);
-      alloc_size += chunk_size;
-    }
-
-    readsize = gzread(gzfile, mem + size, chunk_size);
-    if (readsize > 0) {
-      size += readsize;
+  /* Finalize the Zstd frame. */
+  size_t ret = 1;
+  while (ret != 0) {
+    ZSTD_outBuffer output = {out_buf, out_len, 0};
+    ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end);
+    if (ZSTD_isError(ret)) {
+      break;
     }
-    else {
+    if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
       break;
     }
+    total_written += output.pos;
   }
 
-  gzclose(gzfile);
-
-  if (size == 0) {
-    MEM_freeN(mem);
-    mem = NULL;
-  }
-  else if (alloc_size != size) {
-    mem = MEM_reallocN(mem, size);
-  }
-
-  *r_size = size;
+  MEM_freeN(out_buf);
+  ZSTD_freeCCtx(ctx);
 
-  return mem;
+  return ZSTD_isError(ret) ? 0 : total_written;
 }
 
-#define CHUNK (256 * 1024)
-
-/* gzip byte array from memory and write it to file at certain position.
- * return size of gzip stream.
- */
-size_t BLI_gzip_mem_to_file_at_pos(
-    void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
+size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
 {
-  int ret, flush;
-  unsigned have;
-  z_stream strm;
-  unsigned char out[CHUNK];
-
-  BLI_fseek(file, gz_stream_offset, 0);
-
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  ret = deflateInit(&strm, compression_level);
-  if (ret != Z_OK) {
-    return 0;
-  }
-
-  strm.avail_in = len;
-  strm.next_in = (Bytef *)buf;
-  flush = Z_FINISH;
-
-  do {
-    strm.avail_out = CHUNK;
-    strm.next_out = out;
-    ret = deflate(&strm, flush);
-    if (ret == Z_STREAM_ERROR) {
-      return 0;
-    }
-    have = CHUNK - strm.avail_out;
-    if (fwrite(out, 1, have, file) != have || ferror(file)) {
-      deflateEnd(&strm);
-      return 0;
-    }
-  } while (strm.avail_out == 0);
+  fseek(file, file_offset, SEEK_SET);
 
-  if (strm.avail_in != 0 || ret != Z_STREAM_END) {
-    return 0;
-  }
+  ZSTD_DCtx *ctx = ZSTD_createDCtx();
 
-  deflateEnd(&strm);
-  return (size_t)strm.total_out;
-}
+  size_t in_len = ZSTD_DStreamInSize();
+  void *in_buf = MEM_mallocN(in_len, __func__);
+  ZSTD_inBuffer input = {in_buf, in_len, 0};
 
-/* read and decompress gzip stream from file at certain position to buffer.
- * return size of decompressed data.
- */
-size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
-{
-  int ret;
-  z_stream strm;
-  size_t chunk = 256 * 1024;
-  unsigned char in[CHUNK];
-
-  BLI_fseek(file, gz_stream_offset, 0);
-
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  strm.avail_in = 0;
-  strm.next_in = Z_NULL;
-  ret = inflateInit(&strm);
-  if (ret != Z_OK) {
-    return 0;
-  }
+  ZSTD_outBuffer output = {buf, len, 0};
 
-  do {
-    strm.avail_in = fread(in, 1, chunk, file);
-    strm.next_in = in;
-    if (ferror(file)) {
-      inflateEnd(&strm);
-      return 0;
+  size_t ret = 0;
+  /* Read and decompress chunks of input data until we have enough output. */
+  while (output.pos < output.size && !ZSTD_isError(ret)) {
+    input.size = fread(in_buf, 1, in_len, file);
+    if (input.size == 0) {
+      break;
     }
 
-    do {
-      strm.avail_out = len;
-      strm.next_out = (Bytef *)buf + strm.total_out;
+    /* Consume input data until we run out or have enough output. */
+    input.pos = 0;
+    while (input.pos < input.size && output.pos < output.size) {
+      ret = ZSTD_decompressStream(ctx, &output, &input);
 
-      ret = inflate(&strm, Z_NO_FLUSH);
-      if (ret == Z_STREAM_ERROR) {
-        return 0;
+      if (ZSTD_isError(ret)) {
+        break;
       }
-    } while (strm.avail_out == 0);
+    }
+  }
 
-  } while (ret != Z_STREAM_END);
+  MEM_freeN(in_buf);
+  ZSTD_freeDCtx(ctx);
 
-  inflateEnd(&strm);
-  return (size_t)strm.total_out;
+  return ZSTD_isError(ret) ? 0 : output.pos;
 }
 
-#undef CHUNK
-
 bool BLI_file_magic_is_gzip(const char header[4])
 {
   /* GZIP itself starts with the magic bytes 0x1f 0x8b.
diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c
index 604c9900355..86bd840ce31 100644
--- a/source/blender/sequencer/intern/image_cache.c
+++ b/source/blender/sequencer/intern/image_cache.c
@@ -102,7 +102,7 @@
 /* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */
 #define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf"
 #define DCACHE_IMAGES_PER_FILE 100
-#define DCACHE_CURRENT_VERSION 1
+#define DCACHE_CURRENT_VERSION 2
 #define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
 
 typedef struct DiskCacheHeaderEntry {
@@ -496,24 +496,34 @@ static size_t deflate_imbuf_to_file(ImBuf *ibuf,
                                     int level,
                                     DiskCacheHeaderEntry *header_entry)
 {
-  if (ibuf->rect) {
-    return BLI_gzip_mem_to_file_at_pos(
-        ibuf->rect, header_entry->size_raw, file, header_entry->offset, level);
+  void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float;
+
+  /* Apply compression if wanted, otherwise just write directly to the file. */
+  if (level > 0) {
+    return BLI_file_zstd_from_mem_at_pos(
+        data, header_entry->size_raw, file, header_entry->offset, level);
   }
 
-  return BLI_gzip_mem_to_file_at_pos(
-      ibuf->rect_float, header_entry->size_raw, file, header_entry->offset, level);
+  fseek(file, header_entry->offset, SEEK_SET);
+  return fwrite(data, 1, header_entry->size_raw, file);
 }
 
 static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry)
 {
-  if (ibuf->rect) {
-    return BLI_ungzip_file_to_mem_at_pos(
-        ibuf->rect, header_entry->size_raw, file, header_entry->offset);
+  void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float;
+  char header[4];
+  fseek(file, header_entry->offset, SEEK_SET);
+  if (fread(header, 1, sizeof(header), file) != sizeof(header)) {
+    return 0;
+  }
+
+  /* Check if the data is compressed or raw. */
+  if (BLI_file_magic_is_zstd(header)) {
+    return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset);
   }
 
-  return BLI_ungzip_file_to_mem_at_pos(
-      ibuf->rect_float, header_entry->size_raw, file, header_entry->offset);
+  fseek(file, header_entry->offset, SEEK_SET);
+  return fread(data, 1, header_entry->size_raw, file);
 }
 
 static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header)
-- 
cgit v1.2.3


From 721fad37a1981a564404b5f708a405503dc18f45 Mon Sep 17 00:00:00 2001
From: Lukas Stockner <lukas.stockner@freenet.de>
Date: Sat, 21 Aug 2021 23:31:51 +0200
Subject: Fix Windows builds after Zstandard commits

---
 source/blender/blenloader/intern/writefile.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 6f43fbf1fa0..90d58514eb5 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -78,7 +78,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <zlib.h>
 
 #ifdef WIN32
 #  include "BLI_winstuff.h"
-- 
cgit v1.2.3