Blenloader: New API that will be used for blenloader decentralization

Design Task: T76372

This part of a larger refactoring towards a more extensible architecture
in Blender: T75724

The API is defined in `BLO_read_write.h`. It adds the small data structures
`BlendWriter`, `BlendDataReader`, `BlendLibReader` and `BlendExpander`.
Those contain context about the current read/write operation. Furthermore,
it adds many functions with the prefixes `BLO_write_*`, `BLO_read_*` and
`BLO_expand_*`.

Lib linking and expanding will probably be handled by the more generic libquery
system eventually. The corresponding parts of the API can be removed then.

4 files changed, 482 insertions, 0 deletions

diff --git a/source/blender/blenloader/BLO_read_write.h b/source/blender/blenloader/BLO_read_write.h
new file mode 100644
index 00000000000..7abfa621736
--- /dev/null
+++ b/source/blender/blenloader/BLO_read_write.h
@@ -0,0 +1,207 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup blenloader
+ *
+ * This file contains an API that allows different parts of Blender to define what data is stored
+ * in .blend files.
+ *
+ * Four callbacks have to be provided to fully implement .blend I/O for a piece of data. One of
+ * those is related to file writing and three for file reading. Reading requires multiple
+ * callbacks, due to the way linking between files works.
+ *
+ * Brief description of the individual callbacks:
+ *  - Blend Write: Define which structs and memory buffers are saved.
+ *  - Blend Read Data: Loads structs and memory buffers from file and updates pointers them.
+ *  - Blend Read Lib: Updates pointers to ID data blocks.
+ *  - Blend Expand: Defines which other data blocks should be loaded (possibly from other files).
+ *
+ * Each of these callbacks uses a different API functions.
+ *
+ * Some parts of Blender, e.g. modifiers, don't require you to implement all four callbacks.
+ * Instead only the first two are necessary. The other two are handled by general ID management. In
+ * the future, we might want to get rid of those two callbacks entirely, but for now they are
+ * necessary.
+ */
+
+#ifndef __BLO_READ_WRITE_H__
+#define __BLO_READ_WRITE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct BlendWriter BlendWriter;
+typedef struct BlendDataReader BlendDataReader;
+typedef struct BlendLibReader BlendLibReader;
+typedef struct BlendExpander BlendExpander;
+
+/* Blend Write API
+ * ===============
+ *
+ * Most functions fall into one of two categories. Either they write a DNA struct or a raw memory
+ * buffer to the .blend file.
+ *
+ * It is safe to pass NULL as data_ptr. In this case nothing will be stored.
+ *
+ * DNA Struct Writing
+ * ------------------
+ *
+ * Functions dealing with DNA structs begin with BLO_write_struct_*.
+ *
+ * DNA struct types can be identified in different ways:
+ *  - Run-time Name: The name is provided as const char *.
+ *  - Compile-time Name: The name is provided at compile time. This can be more efficient. Note
+ *      that this optimization is not implemented currently.
+ *  - Struct ID: Every DNA struct type has an integer ID that can be queried with
+ *      BLO_get_struct_id_by_name. Providing this ID can be a useful optimization when many structs
+ *      of the same type are stored AND if those structs are not in a continuous array.
+ *
+ * Often only a single instance of a struct is written at once. However, sometimes it is necessary
+ * to write arrays or linked lists. Separate functions for that are provided as well.
+ *
+ * There is a special macro for writing id structs: BLO_write_id_struct. Those are handled
+ * differently from other structs.
+ *
+ * Raw Data Writing
+ * ----------------
+ *
+ * At the core there is BLO_write_raw, which can write arbitrary memory buffers to the file. The
+ * code that reads this data might have to correct its byte-order. For the common cases there are
+ * convenience functions that write and read arrays of simple types such as int32. Those will
+ * correct endianness automatically.
+ */
+
+/* Mapping between names and ids. */
+int BLO_get_struct_id_by_name(BlendWriter *writer, const char *struct_name);
+#define BLO_get_struct_id(writer, struct_name) BLO_get_struct_id_by_name(writer, #struct_name)
+
+/* Write single struct. */
+void BLO_write_struct_by_name(BlendWriter *writer, const char *struct_name, const void *data_ptr);
+void BLO_write_struct_by_id(BlendWriter *writer, int struct_id, const void *data_ptr);
+#define BLO_write_struct(writer, struct_name, data_ptr) \
+  BLO_write_struct_by_id(writer, BLO_get_struct_id(writer, struct_name), data_ptr)
+
+/* Write struct array. */
+void BLO_write_struct_array_by_name(BlendWriter *writer,
+                                    const char *struct_name,
+                                    int array_size,
+                                    const void *data_ptr);
+void BLO_write_struct_array_by_id(BlendWriter *writer,
+                                  int struct_id,
+                                  int array_size,
+                                  const void *data_ptr);
+#define BLO_write_struct_array(writer, struct_name, array_size, data_ptr) \
+  BLO_write_struct_array_by_id( \
+      writer, BLO_get_struct_id(writer, struct_name), array_size, data_ptr)
+
+/* Write struct list. */
+void BLO_write_struct_list_by_name(BlendWriter *writer,
+                                   const char *struct_name,
+                                   struct ListBase *list);
+void BLO_write_struct_list_by_id(BlendWriter *writer, int struct_id, struct ListBase *list);
+#define BLO_write_struct_list(writer, struct_name, list_ptr) \
+  BLO_write_struct_list_by_id(writer, BLO_get_struct_id(writer, struct_name), list_ptr)
+
+/* Write id struct. */
+void blo_write_id_struct(BlendWriter *writer,
+                         int struct_id,
+                         const void *id_address,
+                         const struct ID *id);
+#define BLO_write_id_struct(writer, struct_name, id_address, id) \
+  blo_write_id_struct(writer, BLO_get_struct_id(writer, struct_name), id_address, id)
+
+/* Write raw data. */
+void BLO_write_raw(BlendWriter *writer, int size_in_bytes, const void *data_ptr);
+void BLO_write_int32_array(BlendWriter *writer, int size, const int32_t *data_ptr);
+void BLO_write_uint32_array(BlendWriter *writer, int size, const uint32_t *data_ptr);
+void BLO_write_float_array(BlendWriter *writer, int size, const float *data_ptr);
+void BLO_write_float3_array(BlendWriter *writer, int size, const float *data_ptr);
+void BLO_write_string(BlendWriter *writer, const char *data_ptr);
+
+/* Misc. */
+bool BLO_write_is_undo(BlendWriter *writer);
+
+/* Blend Read Data API
+ * ===================
+ *
+ * Generally, for every BLO_write_* call there should be a corresponding BLO_read_* call.
+ *
+ * Most BLO_read_* functions get a pointer to a pointer as argument. That allows the function to
+ * update the pointer to its new value.
+ *
+ * When the given pointer points to a memory buffer that was not stored in the file, the pointer is
+ * updated to be NULL. When it was pointing to NULL before, it will stay that way.
+ *
+ * Examples of matching calls:
+ *   BLO_write_struct(writer, ClothSimSettings, clmd->sim_parms);
+ *   BLO_read_data_address(reader, &clmd->sim_parms);
+ *
+ *   BLO_write_struct_list(writer, TimeMarker, &action->markers);
+ *   BLO_read_list(reader, &action->markers, NULL);
+ *
+ *   BLO_write_int32_array(writer, hmd->totindex, hmd->indexar);
+ *   BLO_read_int32_array(reader, hmd->totindex, &hmd->indexar);
+ */
+
+void *BLO_read_get_new_data_address(BlendDataReader *reader, const void *old_address);
+
+#define BLO_read_data_address(reader, ptr_p) \
+  *(ptr_p) = BLO_read_get_new_data_address((reader), *(ptr_p))
+
+typedef void (*BlendReadListFn)(BlendDataReader *reader, void *data);
+void BLO_read_list(BlendDataReader *reader, struct ListBase *list, BlendReadListFn callback);
+
+/* Update data pointers and correct byte-order if necessary. */
+void BLO_read_int32_array(BlendDataReader *reader, int array_size, int32_t **ptr_p);
+void BLO_read_uint32_array(BlendDataReader *reader, int array_size, uint32_t **ptr_p);
+void BLO_read_float_array(BlendDataReader *reader, int array_size, float **ptr_p);
+void BLO_read_float3_array(BlendDataReader *reader, int array_size, float **ptr_p);
+void BLO_read_double_array(BlendDataReader *reader, int array_size, double **ptr_p);
+void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p);
+
+/* Misc. */
+bool BLO_read_requires_endian_switch(BlendDataReader *reader);
+
+/* Blend Read Lib API
+ * ===================
+ *
+ * This API does almost the same as the Blend Read Data API. However, now only pointers to ID data
+ * blocks are updated.
+ */
+
+ID *BLO_read_get_new_id_address(BlendLibReader *reader, struct Library *lib, struct ID *id);
+
+#define BLO_read_id_address(reader, lib, id_ptr_p) \
+  *(id_ptr_p) = (void *)BLO_read_get_new_id_address((reader), (lib), (ID *)*(id_ptr_p))
+
+/* Blend Expand API
+ * ===================
+ *
+ * BLO_expand has to be called for every data block that should be loaded. If the data block is in
+ * a separate .blend file, it will be pulled from there.
+ */
+
+void BLO_expand_id(BlendExpander *expander, struct ID *id);
+
+#define BLO_expand(expander, id) BLO_expand_id(expander, (struct ID *)id)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __BLO_READ_WRITE_H__ */
diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt
index 1555b9231ed..09e2f4bf417 100644
--- a/source/blender/blenloader/CMakeLists.txt
+++ b/source/blender/blenloader/CMakeLists.txt
@@ -64,6 +64,7 @@ set(SRC
   BLO_blend_defs.h
   BLO_blend_validate.h
   BLO_readfile.h
+  BLO_read_write.h
   BLO_undofile.h
   BLO_writefile.h
   intern/readfile.h
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 501e8de678d..53502a8138a 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -160,6 +160,7 @@
 
 #include "BLO_blend_defs.h"
 #include "BLO_blend_validate.h"
+#include "BLO_read_write.h"
 #include "BLO_readfile.h"
 #include "BLO_undofile.h"
 
@@ -713,6 +714,20 @@ static Main *blo_find_main(FileData *fd, const char *filepath, const char *relab
 /** \name File Parsing
  * \{ */
 
+typedef struct BlendDataReader {
+  FileData *fd;
+} BlendDataReader;
+
+typedef struct BlendLibReader {
+  FileData *fd;
+  Main *main;
+} BlendLibReader;
+
+typedef struct BlendExpander {
+  FileData *fd;
+  Main *main;
+} BlendExpander;
+
 static void switch_endian_bh4(BHead4 *bhead)
 {
   /* the ID_.. codes */
@@ -12665,4 +12680,164 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
   BKE_main_free(main_newid);
 }
 
+void *BLO_read_get_new_data_address(BlendDataReader *reader, const void *old_address)
+{
+  return newdataadr(reader->fd, old_address);
+}
+
+ID *BLO_read_get_new_id_address(BlendLibReader *reader, Library *lib, ID *id)
+{
+  return newlibadr(reader->fd, lib, id);
+}
+
+bool BLO_read_requires_endian_switch(BlendDataReader *reader)
+{
+  return (reader->fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
+}
+
+/**
+ * Updates all ->prev and ->next pointers of the list elements.
+ * Updates the list->first and list->last pointers.
+ * When not NULL, calls the callback on every element.
+ */
+void BLO_read_list(BlendDataReader *reader, ListBase *list, BlendReadListFn callback)
+{
+  if (BLI_listbase_is_empty(list)) {
+    return;
+  }
+
+  BLO_read_data_address(reader, &list->first);
+  if (callback != NULL) {
+    callback(reader, list->first);
+  }
+  Link *ln = list->first;
+  Link *prev = NULL;
+  while (ln) {
+    BLO_read_data_address(reader, &ln->next);
+    if (ln->next != NULL && callback != NULL) {
+      callback(reader, ln->next);
+    }
+    ln->prev = prev;
+    prev = ln;
+    ln = ln->next;
+  }
+  list->last = prev;
+}
+
+void BLO_read_int32_array(BlendDataReader *reader, int array_size, int32_t **ptr_p)
+{
+  BLO_read_data_address(reader, ptr_p);
+  if (BLO_read_requires_endian_switch(reader)) {
+    BLI_endian_switch_int32_array(*ptr_p, array_size);
+  }
+}
+
+void BLO_read_uint32_array(BlendDataReader *reader, int array_size, uint32_t **ptr_p)
+{
+  BLO_read_data_address(reader, ptr_p);
+  if (BLO_read_requires_endian_switch(reader)) {
+    BLI_endian_switch_uint32_array(*ptr_p, array_size);
+  }
+}
+
+void BLO_read_float_array(BlendDataReader *reader, int array_size, float **ptr_p)
+{
+  BLO_read_data_address(reader, ptr_p);
+  if (BLO_read_requires_endian_switch(reader)) {
+    BLI_endian_switch_float_array(*ptr_p, array_size);
+  }
+}
+
+void BLO_read_float3_array(BlendDataReader *reader, int array_size, float **ptr_p)
+{
+  BLO_read_float_array(reader, array_size * 3, ptr_p);
+}
+
+void BLO_read_double_array(BlendDataReader *reader, int array_size, double **ptr_p)
+{
+  BLO_read_data_address(reader, ptr_p);
+  if (BLO_read_requires_endian_switch(reader)) {
+    BLI_endian_switch_double_array(*ptr_p, array_size);
+  }
+}
+
+static void convert_pointer_array_64_to_32(BlendDataReader *reader,
+                                           uint array_size,
+                                           const uint64_t *src,
+                                           uint32_t *dst)
+{
+  /* Match pointer conversion rules from bh4_from_bh8 and cast_pointer. */
+  if (BLO_read_requires_endian_switch(reader)) {
+    for (int i = 0; i < array_size; i++) {
+      uint64_t ptr = src[i];
+      BLI_endian_switch_uint64(&ptr);
+      dst[i] = (uint32_t)(ptr >> 3);
+    }
+  }
+  else {
+    for (int i = 0; i < array_size; i++) {
+      dst[i] = (uint32_t)(src[i] >> 3);
+    }
+  }
+}
+
+static void convert_pointer_array_32_to_64(BlendDataReader *UNUSED(reader),
+                                           uint array_size,
+                                           const uint32_t *src,
+                                           uint64_t *dst)
+{
+  /* Match pointer conversion rules from bh8_from_bh4 and cast_pointer. */
+  for (int i = 0; i < array_size; i++) {
+    dst[i] = src[i];
+  }
+}
+
+void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
+{
+  FileData *fd = reader->fd;
+
+  void *orig_array = newdataadr(fd, *ptr_p);
+  if (orig_array == NULL) {
+    *ptr_p = NULL;
+    return;
+  }
+
+  int file_pointer_size = fd->filesdna->pointer_size;
+  int current_pointer_size = fd->memsdna->pointer_size;
+
+  /* Overallocation is fine, but might be better to pass the length as parameter. */
+  int array_size = MEM_allocN_len(orig_array) / file_pointer_size;
+
+  void *final_array = NULL;
+
+  if (file_pointer_size == current_pointer_size) {
+    /* No pointer conversion necessary. */
+    final_array = orig_array;
+  }
+  else if (file_pointer_size == 8 && current_pointer_size == 4) {
+    /* Convert pointers from 64 to 32 bit. */
+    final_array = MEM_malloc_arrayN(array_size, 4, "new pointer array");
+    convert_pointer_array_64_to_32(
+        reader, array_size, (uint64_t *)orig_array, (uint32_t *)final_array);
+    MEM_freeN(orig_array);
+  }
+  else if (file_pointer_size == 4 && current_pointer_size == 8) {
+    /* Convert pointers from 32 to 64 bit. */
+    final_array = MEM_malloc_arrayN(array_size, 8, "new pointer array");
+    convert_pointer_array_32_to_64(
+        reader, array_size, (uint32_t *)orig_array, (uint64_t *)final_array);
+    MEM_freeN(orig_array);
+  }
+  else {
+    BLI_assert(false);
+  }
+
+  *ptr_p = final_array;
+}
+
+void BLO_expand_id(BlendExpander *expander, ID *id)
+{
+  expand_doit(expander->fd, expander->main, id);
+}
+
 /** \} */
diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 70d4d1ba5ed..436ff8fdd58 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -177,6 +177,7 @@
 
 #include "BLO_blend_defs.h"
 #include "BLO_blend_validate.h"
+#include "BLO_read_write.h"
 #include "BLO_readfile.h"
 #include "BLO_undofile.h"
 #include "BLO_writefile.h"
@@ -339,6 +340,10 @@ typedef struct {
   WriteWrap *ww;
 } WriteData;
 
+typedef struct BlendWriter {
+  WriteData *wd;
+} BlendWriter;
+
 static WriteData *writedata_new(WriteWrap *ww)
 {
   WriteData *wd = MEM_callocN(sizeof(*wd), "writedata");
@@ -4536,4 +4541,98 @@ bool BLO_write_file_mem(Main *mainvar, MemFile *compare, MemFile *current, int w
   return (err == 0);
 }
 
+void BLO_write_raw(BlendWriter *writer, int size_in_bytes, const void *data_ptr)
+{
+  writedata(writer->wd, DATA, size_in_bytes, data_ptr);
+}
+
+void BLO_write_struct_by_name(BlendWriter *writer, const char *struct_name, const void *data_ptr)
+{
+  int struct_id = BLO_get_struct_id_by_name(writer, struct_name);
+  BLO_write_struct_by_id(writer, struct_id, data_ptr);
+}
+
+void BLO_write_struct_array_by_name(BlendWriter *writer,
+                                    const char *struct_name,
+                                    int array_size,
+                                    const void *data_ptr)
+{
+  int struct_id = BLO_get_struct_id_by_name(writer, struct_name);
+  BLO_write_struct_array_by_id(writer, struct_id, array_size, data_ptr);
+}
+
+void BLO_write_struct_by_id(BlendWriter *writer, int struct_id, const void *data_ptr)
+{
+  writestruct_nr(writer->wd, DATA, struct_id, 1, data_ptr);
+}
+
+void BLO_write_struct_array_by_id(BlendWriter *writer,
+                                  int struct_id,
+                                  int array_size,
+                                  const void *data_ptr)
+{
+  writestruct_nr(writer->wd, DATA, struct_id, array_size, data_ptr);
+}
+
+void BLO_write_struct_list_by_id(BlendWriter *writer, int struct_id, ListBase *list)
+{
+  writelist_nr(writer->wd, DATA, struct_id, list);
+}
+
+void BLO_write_struct_list_by_name(BlendWriter *writer, const char *struct_name, ListBase *list)
+{
+  BLO_write_struct_list_by_id(writer, BLO_get_struct_id_by_name(writer, struct_name), list);
+}
+
+void blo_write_id_struct(BlendWriter *writer, int struct_id, const void *id_address, const ID *id)
+{
+  writestruct_at_address_nr(writer->wd, GS(id->name), struct_id, 1, id_address, id);
+}
+
+int BLO_get_struct_id_by_name(BlendWriter *writer, const char *struct_name)
+{
+  int struct_id = DNA_struct_find_nr(writer->wd->sdna, struct_name);
+  BLI_assert(struct_id >= 0);
+  return struct_id;
+}
+
+void BLO_write_int32_array(BlendWriter *writer, int size, const int32_t *data_ptr)
+{
+  BLO_write_raw(writer, sizeof(int32_t) * size, data_ptr);
+}
+
+void BLO_write_uint32_array(BlendWriter *writer, int size, const uint32_t *data_ptr)
+{
+  BLO_write_raw(writer, sizeof(uint32_t) * size, data_ptr);
+}
+
+void BLO_write_float_array(BlendWriter *writer, int size, const float *data_ptr)
+{
+  BLO_write_raw(writer, sizeof(float) * size, data_ptr);
+}
+
+void BLO_write_float3_array(BlendWriter *writer, int size, const float *data_ptr)
+{
+  BLO_write_raw(writer, sizeof(float) * 3 * size, data_ptr);
+}
+
+/**
+ * Write a null terminated string.
+ */
+void BLO_write_string(BlendWriter *writer, const char *str)
+{
+  if (str != NULL) {
+    BLO_write_raw(writer, strlen(str) + 1, str);
+  }
+}
+
+/**
+ * Sometimes different data is written depending on whether the file is saved to disk or used for
+ * undo. This function returns true when the current file-writing is done for undo.
+ */
+bool BLO_write_is_undo(BlendWriter *writer)
+{
+  return writer->wd->use_memfile;
+}
+
 /** \} */