DNA: support versioning structs & struct members

This is only to be used rarely because it's not forwards compatible.

Replace version patching of old 2.80 DNA with a more generic API.

9 files changed, 220 insertions, 80 deletions

diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt
index efd64b1a328..b9bec1b97b3 100644
--- a/source/blender/blenloader/CMakeLists.txt
+++ b/source/blender/blenloader/CMakeLists.txt
@@ -54,6 +54,7 @@ set(SRC
 	intern/versioning_270.c
 	intern/versioning_280.c
 	intern/versioning_defaults.c
+	intern/versioning_dna.c
 	intern/versioning_legacy.c
 	intern/versioning_userdef.c
 	intern/writefile.c
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index c4fdb878b86..55a72e363e9 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -932,13 +932,30 @@ static void decode_blender_header(FileData *fd)
 static bool read_file_dna(FileData *fd, const char **r_error_message)
 {
 	BHead *bhead;
+	int subversion = 0;
 
 	for (bhead = blo_firstbhead(fd); bhead; bhead = blo_nextbhead(fd, bhead)) {
-		if (bhead->code == DNA1) {
+		if (bhead->code == GLOB) {
+			/* Before this, the subversion didn't exist in 'FileGlobal' so the subversion
+			 * value isn't accessible for the purpose of DNA versioning in this case. */
+			if (fd->fileversion <= 242) {
+				continue;
+			}
+			/* We can't use read_global because this needs 'DNA1' to be decoded,
+			 * however the first 4 chars are _always_ the subversion. */
+			FileGlobal *fg = (void *)&bhead[1];
+			BLI_STATIC_ASSERT(offsetof(FileGlobal, subvstr) == 0, "Must be first: subvstr");
+			char num[5];
+			memcpy(num, fg->subvstr, 4);
+			num[4] = 0;
+			subversion = atoi(num);
+		}
+		else if (bhead->code == DNA1) {
 			const bool do_endian_swap = (fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
 
 			fd->filesdna = DNA_sdna_from_data(&bhead[1], bhead->len, do_endian_swap, true, r_error_message);
 			if (fd->filesdna) {
+				blo_do_versions_dna(fd->filesdna, fd->fileversion, subversion);
 				fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
 				/* used to retrieve ID names from (bhead+1) */
 				fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h
index 85416892f25..39a474dfe49 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -150,6 +150,8 @@ const char *bhead_id_name(const FileData *fd, const BHead *bhead);
 
 void blo_reportf_wrap(struct ReportList *reports, ReportType type, const char *format, ...) ATTR_PRINTF_FORMAT(3, 4);
 
+void blo_do_versions_dna(struct SDNA *sdna, const int versionfile, const int subversionfile);
+
 void blo_do_versions_oldnewmap_insert(struct OldNewMap *onm, const void *oldaddr, void *newaddr, int nr);
 void *blo_do_versions_newlibadr(struct FileData *fd, const void *lib, const void *adr);
 void *blo_do_versions_newlibadr_us(struct FileData *fd, const void *lib, const void *adr);
diff --git a/source/blender/blenloader/intern/versioning_dna.c b/source/blender/blenloader/intern/versioning_dna.c
new file mode 100644
index 00000000000..d379edd83f6
--- /dev/null
+++ b/source/blender/blenloader/intern/versioning_dna.c
@@ -0,0 +1,59 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file \ingroup blenloader
+ *
+ * Apply edits to DNA at load time
+ * to behave as if old files were written new names.
+ */
+
+#include "BLI_compiler_attrs.h"
+#include "BLI_utildefines.h"
+
+#include "DNA_genfile.h"
+#include "DNA_listBase.h"
+
+#include "BLO_readfile.h"
+#include "readfile.h"
+
+/**
+ * Manipulates SDNA before calling #DNA_struct_get_compareflags,
+ * allowing us to rename structs and struct members.
+ *
+ * \attention Changes here will cause breakages in forward compatbility,
+ * Use this only in the _rare_ cases when migrating to new naming is needed.
+ */
+void blo_do_versions_dna(SDNA *sdna, const int versionfile, const int subversionfile)
+{
+#define DNA_VERSION_ATLEAST(ver, subver) \
+	(versionfile > (ver) || (versionfile == (ver) && (subversionfile >= (subver))))
+
+	if (!DNA_VERSION_ATLEAST(280, 2)) {
+		/* Version files created in the 'blender2.8' branch
+		 * between October 2016, and November 2017 (>=280.0 and < 280.2). */
+		if (versionfile >= 280) {
+			DNA_sdna_patch_struct(sdna, "SceneLayer", "ViewLayer");
+			DNA_sdna_patch_struct(sdna, "SceneLayerEngineData", "ViewLayerEngineData");
+			DNA_sdna_patch_struct_member(sdna, "FileGlobal", "cur_render_layer", "cur_view_layer");
+			DNA_sdna_patch_struct_member(sdna, "ParticleEditSettings", "scene_layer", "view_layer");
+			DNA_sdna_patch_struct_member(sdna, "Scene", "active_layer", "active_view_layer");
+			DNA_sdna_patch_struct_member(sdna, "Scene", "render_layers", "view_layers");
+			DNA_sdna_patch_struct_member(sdna, "WorkSpace", "render_layer", "view_layer");
+		}
+	}
+
+#undef DNA_VERSION_ATLEAST
+}
diff --git a/source/blender/makesdna/DNA_fileglobal_types.h b/source/blender/makesdna/DNA_fileglobal_types.h
index c5833fc3790..52fbdbc22b2 100644
--- a/source/blender/makesdna/DNA_fileglobal_types.h
+++ b/source/blender/makesdna/DNA_fileglobal_types.h
@@ -28,8 +28,9 @@
  * the moment of saving, and the file-specific settings.
  */
 typedef struct FileGlobal {
-	/** Needs to be here, for human fileformat recognition. */
+	/** Needs to be here, for human fileformat recognition (keep first!). */
 	char subvstr[4];
+
 	short subversion;
 	short minversion, minsubversion;
 	char pad[6];
diff --git a/source/blender/makesdna/DNA_genfile.h b/source/blender/makesdna/DNA_genfile.h
index 774b9038bd1..4f67246c891 100644
--- a/source/blender/makesdna/DNA_genfile.h
+++ b/source/blender/makesdna/DNA_genfile.h
@@ -103,4 +103,9 @@ bool DNA_struct_elem_find(const struct SDNA *sdna, const char *stype, const char
 
 int DNA_elem_type_size(const eSDNA_Type elem_nr);
 
+bool DNA_sdna_patch_struct(
+        struct SDNA *sdna, const char *struct_name_old, const char *struct_name_new);
+bool DNA_sdna_patch_struct_member(
+        struct SDNA *sdna, const char *struct_name, const char *member_old, const char *member_new);
+
 #endif /* __DNA_GENFILE_H__ */
diff --git a/source/blender/makesdna/DNA_sdna_types.h b/source/blender/makesdna/DNA_sdna_types.h
index 1adeb3017b5..3ca2ce2fca4 100644
--- a/source/blender/makesdna/DNA_sdna_types.h
+++ b/source/blender/makesdna/DNA_sdna_types.h
@@ -22,6 +22,8 @@
 #ifndef __DNA_SDNA_TYPES_H__
 #define __DNA_SDNA_TYPES_H__
 
+struct MemArena;
+
 #
 #
 typedef struct SDNA {
@@ -59,6 +61,9 @@ typedef struct SDNA {
 
 	/** #GHash for faster lookups, requires WITH_DNA_GHASH to be used for now. */
 	struct GHash *structs_map;
+
+	/** Temporary memory currently only used for version patching DNA. */
+	struct MemArena *mem_arena;
 } SDNA;
 
 #
diff --git a/source/blender/makesdna/intern/CMakeLists.txt b/source/blender/makesdna/intern/CMakeLists.txt
index 4a53b743902..8ad1d189ccf 100644
--- a/source/blender/makesdna/intern/CMakeLists.txt
+++ b/source/blender/makesdna/intern/CMakeLists.txt
@@ -104,6 +104,8 @@ set(INC_SYS
 set(SRC
 	../../blenlib/intern/BLI_ghash.c
 	../../blenlib/intern/BLI_ghash_utils.c
+	../../blenlib/intern/BLI_linklist.c
+	../../blenlib/intern/BLI_memarena.c
 	../../blenlib/intern/BLI_mempool.c
 	../../blenlib/intern/endian_switch.c
 	../../blenlib/intern/hash_mm2a.c
diff --git a/source/blender/makesdna/intern/dna_genfile.c b/source/blender/makesdna/intern/dna_genfile.c
index 650ee29594f..3d0f4fa1aab 100644
--- a/source/blender/makesdna/intern/dna_genfile.c
+++ b/source/blender/makesdna/intern/dna_genfile.c
@@ -36,6 +36,7 @@
 
 #include "BLI_utildefines.h"
 #include "BLI_endian_switch.h"
+#include "BLI_memarena.h"
 
 #ifdef WITH_DNA_GHASH
 #  include "BLI_ghash.h"
@@ -193,6 +194,10 @@ void DNA_sdna_free(SDNA *sdna)
 	}
 #endif
 
+	if (sdna->mem_arena) {
+		BLI_memarena_free(sdna->mem_arena);
+	}
+
 	MEM_freeN(sdna);
 }
 
@@ -328,52 +333,10 @@ BLI_INLINE const char *pad_up_4(const char *ptr)
 }
 
 /**
- * Temporary DNA doversion for files that were created with Blender 2.80
- * between October 2016, and November 2017 (>=280.0 and < 280.2).
- *
- * /note This would be way more efficient if we can get the version from SDNA
- * So we could return true if version == 280 && subversion < 2.
- *
- * Returns true if we need to do the DNA renaming.
- */
-static bool need_doversion_280(SDNA *sdna, int *data, const bool data_alloc)
-{
-	if (data_alloc == false) {
-		return false;
-	}
-
-	bool active_layer = false, render_layers = false;
-
-	const char *cp = (char *)data;
-	for (int nr = 0; nr < sdna->nr_names; nr++) {
-		if (strcmp(cp, "active_layer") == 0) {
-			active_layer = true;
-			if (active_layer && render_layers) {
-				return true;
-			}
-		}
-		else if (strcmp(cp, "render_layers") == 0) {
-			render_layers = true;
-			if (active_layer && render_layers) {
-				return true;
-			}
-		}
-
-		while (*cp) cp++;
-		cp++;
-	}
-
-	/* If someone adds only one of them to the DNA, don't! */
-	BLI_assert(!(active_layer || render_layers));
-	return false;
-}
-
-/**
  * In sdna->data the data, now we convert that to something understandable
  */
 static bool init_structDNA(
         SDNA *sdna, bool do_endian_swap,
-        bool data_alloc,
         const char **r_error_message)
 {
 	int *data, gravity_fix = -1;
@@ -388,6 +351,7 @@ static bool init_structDNA(
 #ifdef WITH_DNA_GHASH
 	sdna->structs_map = NULL;
 #endif
+	sdna->mem_arena = NULL;
 
 	/* Struct DNA ('SDNA') */
 	if (*data != MAKE_ID('S', 'D', 'N', 'A')) {
@@ -415,10 +379,6 @@ static bool init_structDNA(
 			return false;
 		}
 
-		/* Temporary DNA doversion for files that were created with Blender 2.80
-		 * between 280.0 and 280.2. */
-		const bool doversion_280 = need_doversion_280(sdna, data, data_alloc);
-
 		cp = (char *)data;
 		for (int nr = 0; nr < sdna->nr_names; nr++) {
 			sdna->names[nr] = cp;
@@ -432,29 +392,6 @@ static bool init_structDNA(
 					gravity_fix = nr;
 				}
 			}
-			else if (doversion_280) {
-				if (strcmp(cp, "*render_layer") == 0) {
-					/* WorkSpace. */
-					sdna->names[nr] = "*view_layer";
-				}
-				else if (strcmp(cp, "*scene_layer") == 0) {
-					/* ParticleEditSettings. */
-					sdna->names[nr] = "*view_layer";
-				}
-				else if (strcmp(cp, "render_layers") == 0) {
-					/* Scene. */
-					sdna->names[nr] = "view_layers";
-				}
-				else if (strcmp(cp, "active_layer") == 0) {
-					/* Scene. */
-					sdna->names[nr] = "active_view_layer";
-				}
-				else if (strcmp(cp, "*cur_render_layer") == 0) {
-					/* FileGlobal. */
-					sdna->names[nr] = "*cur_view_layer";
-				}
-			}
-
 			while (*cp) cp++;
 			cp++;
 		}
@@ -498,14 +435,6 @@ static bool init_structDNA(
 			else if (strcmp("CollectionObject", cp) == 0) {
 				sdna->types[nr] = "GroupObject";
 			}
-			else if (doversion_280) {
-				if (strcmp(cp, "SceneLayer") == 0) {
-					sdna->types[nr] = "ViewLayer";
-				}
-				else if (strcmp(cp, "SceneLayerEngineData") == 0) {
-					sdna->types[nr] = "ViewLayerEngineData";
-				}
-			}
 
 			while (*cp) cp++;
 			cp++;
@@ -646,7 +575,7 @@ SDNA *DNA_sdna_from_data(
 	sdna->data_alloc = data_alloc;
 
 
-	if (init_structDNA(sdna, do_endian_swap, data_alloc, &error_message)) {
+	if (init_structDNA(sdna, do_endian_swap, &error_message)) {
 		return sdna;
 	}
 	else {
@@ -1475,3 +1404,122 @@ int DNA_elem_type_size(const eSDNA_Type elem_nr)
 	/* weak */
 	return 8;
 }
+
+/* -------------------------------------------------------------------- */
+/** \name Version Patch DNA
+ * \{ */
+
+static bool is_identifier(const char c)
+{
+	return ((c >= 'a' && c <= 'z') ||
+	        (c >= 'A' && c <= 'Z') ||
+	        (c >= '0' && c <= '9') ||
+	        (c == '_'));
+}
+
+/**
+ * Check if 'var' matches '*var[3]' for eg,
+ * return true if it does, with start/end offsets.
+ */
+static char str_member_match(
+        const char *member_search, const int member_search_len,
+        const char *member_dna,
+        uint *r_member_dna_offset)
+{
+	BLI_assert(strlen(member_search) == member_search_len);
+	int member_dna_offset = 0;
+	while (!is_identifier(member_dna[member_dna_offset])) {
+		member_dna_offset++;
+	}
+	const char *member_dna_trim = member_dna + member_dna_offset;
+	if (strncmp(member_search, member_dna_trim, member_search_len) == 0) {
+		const char c = member_dna_trim[member_search_len];
+		if (c == '\0' || !is_identifier(c)) {
+			*r_member_dna_offset = member_dna_offset;
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool DNA_sdna_patch_struct_nr(
+        SDNA *sdna, const int struct_name_old_nr, const char *struct_name_new)
+{
+	BLI_assert(DNA_struct_find_nr(DNA_sdna_current_get(), struct_name_new) != -1);
+	const short *sp = sdna->structs[struct_name_old_nr];
+#ifdef WITH_DNA_GHASH
+	BLI_ghash_remove(sdna->structs_map, (void *)sdna->types[sp[0]], NULL, NULL);
+	BLI_ghash_insert(sdna->structs_map, (void *)struct_name_new, POINTER_FROM_INT(struct_name_old_nr));
+#endif
+	// printf("Struct rename: %s -> %s\n", sdna->types[struct_name_old_nr], struct_name_new);
+	sdna->types[sp[0]] = struct_name_new;
+	return true;
+}
+/**
+ * Rename a struct
+ */
+bool DNA_sdna_patch_struct(
+        SDNA *sdna, const char *struct_name_old, const char *struct_name_new)
+{
+	const int struct_name_old_nr = DNA_struct_find_nr(sdna, struct_name_old);
+	if (struct_name_old_nr != -1) {
+		return DNA_sdna_patch_struct_nr(sdna, struct_name_old_nr, struct_name_new);
+	}
+	return false;
+}
+
+/* Make public if called often with same struct (avoid duplicate look-ups). */
+static bool DNA_sdna_patch_struct_member_nr(
+        SDNA *sdna, const int struct_name_nr, const char *member_old, const char *member_new)
+{
+	const int member_old_len = strlen(member_old);
+	const int member_new_len = strlen(member_new);
+	BLI_assert(member_new != NULL);
+	const short *sp = sdna->structs[struct_name_nr];
+	for (int member_iter = sp[1]; member_iter > 0; member_iter--, sp += 2) {
+		const char *member_dna_old = sdna->names[sp[1]];
+		/* Start & end offsets in 'member_dna_old'. */
+		uint member_dna_offset_start;
+		if (str_member_match(member_old, member_old_len, member_dna_old, &member_dna_offset_start)) {
+			if (sdna->mem_arena == NULL) {
+				sdna->mem_arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
+			}
+			const int member_dna_old_len = strlen(member_dna_old);
+			const int member_final_len = (member_dna_old_len - member_old_len) + member_new_len;
+			char *member_dna_new = BLI_memarena_alloc(sdna->mem_arena, member_final_len + 1);
+			uint i = 0;
+			if (member_dna_offset_start != 0) {
+				memcpy(member_dna_new, member_dna_old, member_dna_offset_start);
+				i = member_dna_offset_start;
+			}
+			memcpy(&member_dna_new[i], member_new, member_new_len + 1);
+			i += member_new_len;
+			uint member_dna_offset_end = member_dna_offset_start + member_old_len;
+			if (member_dna_old[member_dna_offset_end] != '\0') {
+				const int member_dna_tail_len = (member_dna_old_len - member_dna_offset_end);
+				memcpy(&member_dna_new[i], &member_dna_old[member_dna_offset_end], member_dna_tail_len + 1);
+				i += member_dna_tail_len;
+			}
+			BLI_assert((strlen(member_dna_new) == member_final_len) && (i == member_final_len));
+			// printf("Struct member rename: '%s.%s' -> '%s'\n", member_old, member_dna_old, member_dna_new);
+			sdna->names[sp[1]] = member_dna_new;
+			return true;
+		}
+	}
+	return false;
+}
+/**
+ * Replace \a member_old with \a member_new for struct \a struct_name
+ * handles search & replace, maintaining surrounding non-identifier characters such as pointer & array size.
+ */
+bool DNA_sdna_patch_struct_member(
+        SDNA *sdna, const char *struct_name, const char *member_old, const char *member_new)
+{
+	const int struct_name_nr = DNA_struct_find_nr(sdna, struct_name);
+	if (struct_name_nr != -1) {
+		return DNA_sdna_patch_struct_member_nr(sdna, struct_name_nr, member_old, member_new);
+	}
+	return false;
+}
+
+/** \} */