From 783f3376eebecd25a8aef6a3697643166ff72162 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Tue, 12 Feb 2019 11:43:33 +1100 Subject: Reviewers: brecht mont29 sergey dflinto Differential Revision: https://developer.blender.org/D4342 --- source/blender/blenloader/intern/versioning_dna.c | 36 +++++-- source/blender/makesdna/DNA_genfile.h | 1 + source/blender/makesdna/intern/CMakeLists.txt | 4 + source/blender/makesdna/intern/dna_genfile.c | 116 ++++++++++++++++++++++ source/blender/makesdna/intern/dna_utils.c | 97 ++++++++++++++++++ source/blender/makesdna/intern/dna_utils.h | 14 +++ source/blender/makesdna/intern/makesdna.c | 89 +++++++++++++++-- source/blender/makesrna/intern/rna_define.c | 38 ++++++- 8 files changed, 378 insertions(+), 17 deletions(-) diff --git a/source/blender/blenloader/intern/versioning_dna.c b/source/blender/blenloader/intern/versioning_dna.c index d379edd83f6..c855971d131 100644 --- a/source/blender/blenloader/intern/versioning_dna.c +++ b/source/blender/blenloader/intern/versioning_dna.c @@ -16,19 +16,27 @@ /** \file \ingroup blenloader * - * Apply edits to DNA at load time - * to behave as if old files were written new names. + * Apply edits to DNA at load time. + * + * - #DNA_MAKESDNA undefined: + * Behave as if old files were written new names. + * - #DNA_MAKESDNA defined: + * Defines store files with old names but runtime uses new names. */ -#include "BLI_compiler_attrs.h" -#include "BLI_utildefines.h" +#ifndef DNA_MAKESDNA +# include "BLI_compiler_attrs.h" +# include "BLI_utildefines.h" + +# include "DNA_genfile.h" +# include "DNA_listBase.h" -#include "DNA_genfile.h" -#include "DNA_listBase.h" +# include "BLO_readfile.h" +# include "readfile.h" +#endif -#include "BLO_readfile.h" -#include "readfile.h" +#ifndef DNA_MAKESDNA /** * Manipulates SDNA before calling #DNA_struct_get_compareflags, * allowing us to rename structs and struct members. @@ -57,3 +65,15 @@ void blo_do_versions_dna(SDNA *sdna, const int versionfile, const int subversion #undef DNA_VERSION_ATLEAST } + +#else /* !DNA_MAKESDNA */ + +/* Included in DNA versioning code. */ + +DNA_STRUCT_REPLACE(Lamp, Light) +DNA_STRUCT_MEMBER_REPLACE(Lamp, clipsta, clip_start) +DNA_STRUCT_MEMBER_REPLACE(Lamp, clipend, clip_end) + +DNA_STRUCT_MEMBER_REPLACE(Camera, YF_dofdist, dof_dist) + +#endif /* !DNA_MAKESDNA */ diff --git a/source/blender/makesdna/DNA_genfile.h b/source/blender/makesdna/DNA_genfile.h index 0d330c8999a..b46ec446791 100644 --- a/source/blender/makesdna/DNA_genfile.h +++ b/source/blender/makesdna/DNA_genfile.h @@ -90,6 +90,7 @@ void DNA_sdna_current_free(void); int DNA_struct_find_nr_ex(const struct SDNA *sdna, const char *str, unsigned int *index_last); int DNA_struct_find_nr(const struct SDNA *sdna, const char *str); + void DNA_struct_switch_endian(const struct SDNA *oldsdna, int oldSDNAnr, char *data); const char *DNA_struct_get_compareflags(const struct SDNA *sdna, const struct SDNA *newsdna); void *DNA_struct_reconstruct( diff --git a/source/blender/makesdna/intern/CMakeLists.txt b/source/blender/makesdna/intern/CMakeLists.txt index 4d6ad4b4389..d6ad251d373 100644 --- a/source/blender/makesdna/intern/CMakeLists.txt +++ b/source/blender/makesdna/intern/CMakeLists.txt @@ -35,7 +35,11 @@ blender_include_dirs( set(SRC dna_utils.c makesdna.c + ../../blenlib/intern/BLI_ghash.c + ../../blenlib/intern/BLI_ghash_utils.c ../../blenlib/intern/BLI_memarena.c + ../../blenlib/intern/BLI_mempool.c + ../../blenlib/intern/hash_mm2a.c ../../../../intern/guardedalloc/intern/mallocn.c ../../../../intern/guardedalloc/intern/mallocn_guarded_impl.c ../../../../intern/guardedalloc/intern/mallocn_lockfree_impl.c diff --git a/source/blender/makesdna/intern/dna_genfile.c b/source/blender/makesdna/intern/dna_genfile.c index 6d4eab11c0f..ed5b00f1458 100644 --- a/source/blender/makesdna/intern/dna_genfile.c +++ b/source/blender/makesdna/intern/dna_genfile.c @@ -156,6 +156,9 @@ void DNA_sdna_free(SDNA *sdna) BLI_memarena_free(sdna->mem_arena); } + MEM_SAFE_FREE(sdna->runtime.names); + MEM_SAFE_FREE(sdna->runtime.types); + MEM_freeN(sdna); } @@ -311,6 +314,9 @@ static bool init_structDNA( #endif sdna->mem_arena = NULL; + /* Lazy initialize. */ + memset(&sdna->runtime, 0, sizeof(sdna->runtime)); + /* Struct DNA ('SDNA') */ if (*data != MAKE_ID('S', 'D', 'N', 'A')) { *r_error_message = "SDNA error in SDNA file"; @@ -1438,3 +1444,113 @@ bool DNA_sdna_patch_struct_member( } /** \} */ + + +/* -------------------------------------------------------------------- */ +/** \name Versioning (Forward Compatible) + * + * Versioning that allows new names. + * \{ */ + +/** + * Unique names are shared, which causes problems renaming. + * Make sure every struct member gets it's own name so any renaming + */ +static void sdna_softpatch_runtime_expand_names(SDNA *sdna) +{ + int names_len_all = 0; + for (int struct_nr = 0; struct_nr < sdna->nr_structs; struct_nr++) { + const short *sp = sdna->structs[struct_nr]; + names_len_all += sp[1]; + } + const char **names_expand = MEM_mallocN(sizeof(*names_expand) * names_len_all, __func__); + + int names_expand_index = 0; + for (int struct_nr = 0; struct_nr < sdna->nr_structs; struct_nr++) { + /* We can't edit this memory 'sdna->structs' points to (readonly datatoc file). */ + const short *sp = sdna->structs[struct_nr]; + short *sp_expand = BLI_memarena_alloc(sdna->mem_arena, sizeof(short[2]) * (1 + sp[1])); + memcpy(sp_expand, sp, sizeof(short[2]) * (1 + sp[1])); + sdna->structs[struct_nr] = sp_expand; + const int names_len = sp[1]; + sp += 2; + sp_expand += 2; + for (int i = 0; i < names_len; i++, sp += 2, sp_expand += 2) { + names_expand[names_expand_index] = sdna->names[sp[1]]; + BLI_assert(names_expand_index < SHRT_MAX); + sp_expand[1] = names_expand_index; + names_expand_index++; + } + } + MEM_freeN((void *)sdna->names); + sdna->names = names_expand; + sdna->nr_names = names_len_all; +} + +void DNA_sdna_softpatch_runtime_ensure(SDNA *sdna) +{ + if (sdna->mem_arena == NULL) { + sdna->mem_arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__); + } + + GHash *struct_map_runtime_from_static; + GHash *elem_map_runtime_from_static; + + DNA_softupdate_maps( + DNA_VERSION_RUNTIME_FROM_STATIC, + &struct_map_runtime_from_static, + &elem_map_runtime_from_static); + + + if (sdna->runtime.types == NULL) { + sdna->runtime.types = MEM_mallocN(sizeof(*sdna->runtime.types) * sdna->nr_types, __func__); + for (int type_nr = 0; type_nr < sdna->nr_types; type_nr++) { + const char *str = sdna->types[type_nr]; + sdna->runtime.types[type_nr] = BLI_ghash_lookup_default( + struct_map_runtime_from_static, str, (void *)str); + } + } + + if (sdna->runtime.names == NULL) { + sdna_softpatch_runtime_expand_names(sdna); + sdna->runtime.names = MEM_mallocN(sizeof(*sdna->runtime.names) * sdna->nr_names, __func__); + for (int struct_nr = 0; struct_nr < sdna->nr_structs; struct_nr++) { + const short *sp = sdna->structs[struct_nr]; + const char *struct_name_static = sdna->types[sp[0]]; + const int dna_struct_names_len = sp[1]; + sp += 2; + for (int a = 0; a < dna_struct_names_len; a++, sp += 2) { + const char *elem_static = sdna->names[sp[1]]; + const uint elem_static_offset_start = DNA_elem_id_offset_start(elem_static); + const char *elem_static_trim = elem_static + elem_static_offset_start; + const uint member_dna_offset_end = ( + elem_static_offset_start + DNA_elem_id_offset_end(elem_static_trim)); + + const uint elem_static_buf_len = member_dna_offset_end - elem_static_offset_start; + char elem_static_buf[1024]; + strncpy(elem_static_buf, elem_static_trim, elem_static_buf_len); + elem_static_buf[elem_static_buf_len] = '\0'; + // printf("Searching '%s.%s'\n", struct_name_static, elem_static_buf); + const char *str_pair[2] = {struct_name_static, elem_static_buf}; + const char *elem_runtime = BLI_ghash_lookup(elem_map_runtime_from_static, str_pair); + if (elem_runtime) { + // printf("Found %s from %s\n", elem_runtime, elem_static_buf); + sdna->runtime.names[sp[1]] = DNA_elem_id_rename( + sdna->mem_arena, + elem_static_trim, strlen(elem_static_trim), + elem_runtime, strlen(elem_runtime), + elem_static, strlen(elem_static), + elem_static_offset_start); + // printf("result is: %s\n", sdna->runtime.names[sp[1]]); + } + else { + sdna->runtime.names[sp[1]] = sdna->names[sp[1]]; + } + } + } + } + BLI_ghash_free(struct_map_runtime_from_static, NULL, NULL); + BLI_ghash_free(elem_map_runtime_from_static, MEM_freeN, NULL); +} + +/** \} */ diff --git a/source/blender/makesdna/intern/dna_utils.c b/source/blender/makesdna/intern/dna_utils.c index f1a04eae7aa..1ca290e73db 100644 --- a/source/blender/makesdna/intern/dna_utils.c +++ b/source/blender/makesdna/intern/dna_utils.c @@ -23,9 +23,12 @@ #include +#include "MEM_guardedalloc.h" + #include "BLI_sys_types.h" #include "BLI_utildefines.h" #include "BLI_assert.h" +#include "BLI_ghash.h" #include "BLI_memarena.h" @@ -104,6 +107,7 @@ uint DNA_elem_id_offset_end(const char *elem_dna) return elem_dna_offset; } + /** * Check if 'var' matches '*var[3]' for eg, * return true if it does, with start/end offsets. @@ -162,3 +166,96 @@ char *DNA_elem_id_rename( } /** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Versioning + * \{ */ + +/* Use for #DNA_VERSIONING_INCLUDE */ +#define DNA_MAKESDNA + +static void dna_softupdate_ghash_add_pair(GHash *gh, const char *a, const char *b, void *value) +{ + const char **str_pair = MEM_mallocN(sizeof(char *) * 2, __func__); + str_pair[0] = a; + str_pair[1] = b; + BLI_ghash_insert(gh, str_pair, value); +} + +static uint strhash_pair_p(const void *ptr) +{ + const char * const *pair = ptr; + return (BLI_ghashutil_strhash_p(pair[0]) ^ + BLI_ghashutil_strhash_p(pair[1])); +} + +static bool strhash_pair_cmp(const void *a, const void *b) +{ + const char * const *pair_a = a; + const char * const *pair_b = b; + return (STREQ(pair_a[0], pair_b[0]) && + STREQ(pair_a[1], pair_b[1])) ? false : true; +} + +void DNA_softupdate_maps( + enum eDNAVersionDir version_dir, + GHash **r_struct_map, GHash **r_elem_map) +{ + + if (r_struct_map != NULL) { + const char *data[][2] = { +#define DNA_STRUCT_REPLACE(old, new) {#old, #new}, +#define DNA_STRUCT_MEMBER_REPLACE(struct_name, old, new) +#include DNA_VERSIONING_DEFINES +#undef DNA_STRUCT_REPLACE +#undef DNA_STRUCT_MEMBER_REPLACE + }; + + int elem_key, elem_val; + if (version_dir == DNA_VERSION_RUNTIME_FROM_STATIC) { + elem_key = 0; + elem_val = 1; + + } + else { + elem_key = 1; + elem_val = 0; + } + + GHash *struct_map = BLI_ghash_str_new_ex(__func__, ARRAY_SIZE(data)); + for (int i = 0; i < ARRAY_SIZE(data); i++) { + BLI_ghash_insert(struct_map, (void *)data[i][elem_key], (void *)data[i][elem_val]); + } + *r_struct_map = struct_map; + } + + if (r_elem_map != NULL) { + const char *data[][3] = { +#define DNA_STRUCT_REPLACE(old, new) +#define DNA_STRUCT_MEMBER_REPLACE(struct_name, old, new) {#struct_name, #old, #new}, +#include DNA_VERSIONING_DEFINES +#undef DNA_STRUCT_REPLACE +#undef DNA_STRUCT_MEMBER_REPLACE + }; + + int elem_key, elem_val; + if (version_dir == DNA_VERSION_RUNTIME_FROM_STATIC) { + elem_key = 1; + elem_val = 2; + + } + else { + elem_key = 2; + elem_val = 1; + } + GHash *elem_map = BLI_ghash_new_ex(strhash_pair_p, strhash_pair_cmp, __func__, ARRAY_SIZE(data)); + for (int i = 0; i < ARRAY_SIZE(data); i++) { + dna_softupdate_ghash_add_pair(elem_map, data[i][0], data[i][elem_key], (void *)data[i][elem_val]); + } + *r_elem_map = elem_map; + } +} + +#undef DNA_MAKESDNA + +/** \} */ diff --git a/source/blender/makesdna/intern/dna_utils.h b/source/blender/makesdna/intern/dna_utils.h index 38fce41440a..c1c0803237b 100644 --- a/source/blender/makesdna/intern/dna_utils.h +++ b/source/blender/makesdna/intern/dna_utils.h @@ -21,6 +21,7 @@ #define __DNA_UTILS_H__ struct MemArena; +struct GHash; int DNA_elem_array_size(const char *str); @@ -37,4 +38,17 @@ char *DNA_elem_id_rename( const char *elem_dna_src, const int elem_dna_src_len, const uint elem_dna_offset_start); +/* When requesting version info, support both directions. */ +enum eDNAVersionDir { + DNA_VERSION_STATIC_FROM_RUNTIME = -1, + DNA_VERSION_RUNTIME_FROM_STATIC = 1, +}; + +void DNA_softupdate_maps( + enum eDNAVersionDir version_dir, + struct GHash **r_struct_map, struct GHash **r_elem_map); + +/* Needs 'DNA_MAKESDNA' to be defined. */ +#define DNA_VERSIONING_DEFINES "../../blenloader/intern/versioning_dna.c" + #endif /* __DNA_UTILS_H__ */ diff --git a/source/blender/makesdna/intern/makesdna.c b/source/blender/makesdna/intern/makesdna.c index e2616d91b27..e2b1d6f3ee0 100644 --- a/source/blender/makesdna/intern/makesdna.c +++ b/source/blender/makesdna/intern/makesdna.c @@ -43,11 +43,15 @@ #include #include #include +#include #include "MEM_guardedalloc.h" -#include "BLI_sys_types.h" /* for intptr_t support */ +#include "BLI_utildefines.h" +#include "BLI_alloca.h" +#include "BLI_ghash.h" #include "BLI_memarena.h" +#include "BLI_sys_types.h" /* for intptr_t support */ #include "dna_utils.h" @@ -153,6 +157,16 @@ static short *typelens_64; * sp[1] is amount of elements * sp[2] sp[3] is typenr, namenr (etc) */ static short **structs, *structdata; + +/** Versioning data */ +static struct { + GHash *struct_map_runtime_from_static; + GHash *struct_map_static_from_runtime; + + GHash *elem_map_runtime_from_static; + GHash *elem_map_static_from_runtime; +} g_version_data = {NULL}; + /** * Variable to control debug output of makesdna. * debugSDNA: @@ -167,10 +181,8 @@ static int additional_slen_offset; #define DEBUG_PRINTF(debug_level, ...) \ { if (debugSDNA > debug_level) { printf(__VA_ARGS__); } } ((void)0) - /* stub for BLI_abort() */ #ifndef NDEBUG -void BLI_system_backtrace(FILE *fp); void BLI_system_backtrace(FILE *fp) { (void)fp; @@ -234,6 +246,51 @@ void printStructLengths(void); /* ************************* MAKEN DNA ********************** */ + +static const char *version_struct_static_from_runtime(const char *str) +{ + const char *str_test = BLI_ghash_lookup(g_version_data.struct_map_static_from_runtime, str); + if (str_test != NULL) { + return str_test; + } + return str; +} + +static const char *version_struct_runtime_from_static(const char *str) +{ + const char *str_test = BLI_ghash_lookup(g_version_data.struct_map_runtime_from_static, str); + if (str_test != NULL) { + return str_test; + } + return str; +} + +static const char *version_struct_elem_runtime_from_static( + const int strct, const char *elem_static_untrimmed) +{ + /* First get the old name with everything stripped out of it. */ + const uint elem_static_offset_start = DNA_elem_id_offset_start(elem_static_untrimmed); + const char *elem_static_trim = elem_static_untrimmed + elem_static_offset_start; + const uint elem_static_len = DNA_elem_id_offset_end(elem_static_trim); + + char *elem_static = alloca(elem_static_len + 1); + memcpy(elem_static, elem_static_trim, elem_static_len); + elem_static[elem_static_len] = '\0'; + + /* Now we have name, no junk around it, get the renamed version. */ + const char *str_pair[2] = {types[strct], elem_static}; + const char *elem_runtime = BLI_ghash_lookup(g_version_data.elem_map_static_from_runtime, str_pair); + if (elem_runtime != NULL) { + return DNA_elem_id_rename( + mem_arena, + elem_static, strlen(elem_static), + elem_runtime, strlen(elem_runtime), + elem_static, elem_static_len, + elem_static_offset_start); + } + return elem_static_untrimmed; +} + static int add_type(const char *str, int len) { int nr; @@ -249,6 +306,8 @@ static int add_type(const char *str, int len) return -1; } + str = version_struct_static_from_runtime(str); + /* search through type array */ for (nr = 0; nr < nr_types; nr++) { if (strcmp(str, types[nr]) == 0) { @@ -668,8 +727,9 @@ static int convert_include(const char *filename) if (md1[slen - 1] == ';') { md1[slen - 1] = 0; + const char *md1_maybe_versioned = version_struct_elem_runtime_from_static(strct, md1); - name = add_name(md1); + name = add_name(md1_maybe_versioned); slen += additional_slen_offset; sp[0] = type; sp[1] = name; @@ -685,8 +745,9 @@ static int convert_include(const char *filename) break; } + const char *md1_maybe_versioned = version_struct_elem_runtime_from_static(strct, md1); - name = add_name(md1); + name = add_name(md1_maybe_versioned); slen += additional_slen_offset; sp[0] = type; @@ -1000,6 +1061,16 @@ static int make_structDNA(const char *baseDirectory, FILE *file, FILE *file_offs typelens_64 = MEM_callocN(sizeof(short) * maxnr, "typelens_64"); structs = MEM_callocN(sizeof(short *) * maxnr, "structs"); + /* Build versioning data */ + DNA_softupdate_maps( + DNA_VERSION_RUNTIME_FROM_STATIC, + &g_version_data.struct_map_runtime_from_static, + &g_version_data.elem_map_runtime_from_static); + DNA_softupdate_maps( + DNA_VERSION_STATIC_FROM_RUNTIME, + &g_version_data.struct_map_static_from_runtime, + &g_version_data.elem_map_static_from_runtime); + /** * Insertion of all known types. * @@ -1182,7 +1253,7 @@ static int make_structDNA(const char *baseDirectory, FILE *file, FILE *file_offs for (i = 0; i < nr_structs; i++) { const short *structpoin = structs[i]; const int structtype = structpoin[0]; - fprintf(file_offsets, "\t_SDNA_TYPE_%s = %d,\n", types[structtype], i); + fprintf(file_offsets, "\t_SDNA_TYPE_%s = %d,\n", version_struct_runtime_from_static(types[structtype]), i); } fprintf(file_offsets, "\tSDNA_TYPE_MAX = %d,\n", nr_structs); fprintf(file_offsets, "};\n"); @@ -1198,6 +1269,12 @@ static int make_structDNA(const char *baseDirectory, FILE *file, FILE *file_offs BLI_memarena_free(mem_arena); + BLI_ghash_free(g_version_data.struct_map_runtime_from_static, NULL, NULL); + BLI_ghash_free(g_version_data.struct_map_static_from_runtime, NULL, NULL); + + BLI_ghash_free(g_version_data.elem_map_runtime_from_static, MEM_freeN, NULL); + BLI_ghash_free(g_version_data.elem_map_static_from_runtime, MEM_freeN, NULL); + DEBUG_PRINTF(0, "done.\n"); return(0); diff --git a/source/blender/makesrna/intern/rna_define.c b/source/blender/makesrna/intern/rna_define.c index 5c44e9b24d9..4c074e189bf 100644 --- a/source/blender/makesrna/intern/rna_define.c +++ b/source/blender/makesrna/intern/rna_define.c @@ -58,6 +58,13 @@ BlenderDefRNA DefRNA = {NULL, {NULL, NULL}, {NULL, NULL}, NULL, 0, 0, 0, 1, 1}; +static struct { + char _dont_leave_me_empty; +// #ifndef RNA_RUNTIME + GHash *struct_map_static_from_runtime; +// #endif +} g_version_data; + /* Duplicated code since we can't link in blenkernel or blenlib */ /* pedantic check for final '.', note '...' are allowed though. */ @@ -354,7 +361,17 @@ static int rna_find_sdna_member(SDNA *sdna, const char *structname, const char * const short *sp; int a, b, structnr, totmember, cmp; - structnr = DNA_struct_find_nr(sdna, structname); + /* TODO: (campbell) either support this by allocating 'struct_map_static_from_runtime' + * at 'RNA_RUNTIME', or disable this function at runtime. */ +#ifdef RNA_RUNTIME + BLI_assert(0); +#endif + { + const char *structname_maybe_static = BLI_ghash_lookup_default( + g_version_data.struct_map_static_from_runtime, structname, (void *)structname); + structnr = DNA_struct_find_nr(sdna, structname_maybe_static); + } + if (structnr == -1) return 0; @@ -363,8 +380,7 @@ static int rna_find_sdna_member(SDNA *sdna, const char *structname, const char * sp += 2; for (a = 0; a < totmember; a++, sp += 2) { - dnaname = sdna->names[sp[1]]; - + dnaname = sdna->runtime.names[sp[1]]; cmp = rna_member_cmp(dnaname, membername); if (cmp == 1) { @@ -572,6 +588,16 @@ BlenderRNA *RNA_create(void) DefRNA.error = 1; } + /* We need runtime and on-disk names. */ + DNA_sdna_softpatch_runtime_ensure(DefRNA.sdna); + +#ifndef RNA_RUNTIME + DNA_softupdate_maps( + DNA_VERSION_STATIC_FROM_RUNTIME, + &g_version_data.struct_map_static_from_runtime, + NULL); +#endif + return brna; } @@ -705,6 +731,12 @@ void RNA_free(BlenderRNA *brna) RNA_struct_free(brna, srna); } } + +#ifndef RNA_RUNTIME + BLI_ghash_free(g_version_data.struct_map_static_from_runtime, NULL, NULL); + g_version_data.struct_map_static_from_runtime = NULL; +#endif + } static size_t rna_property_type_sizeof(PropertyType type) -- cgit v1.2.3