AssetsBrowser: Add ID Properties to Asset Indexer

Object/collection asset workflow would need the bounding box for snapping.
The bounding box is stored using ID properties in the scene. Currently ID properties
aren't stored in the asset index, what would break object snapping. For this reason
Asset Indexing is turned off in mater. This patch will introduce the indexing of ID
properties what will allow the indexing to be turned on again.

## Data Mapping ##

For data mapping we store the internal structure of IDProperty to the indexer (including meta-data) to be able to deserialize it back.
```
[
  {
    "name":  ..,
    "value": ..,
    "type": ..,
    /* `subtype` and `length` are only available for IDP_ARRAYs. */
    "subtype": ..,
  },
]
```

| **DNA** | **Serialize type** | **Note** |
| IDProperty.name | StringValue| |
| IDProperty.type | StringValue| "IDP_STRING", "IDP_INT", "IDP_FLOAT", "IDP_ARRAY", "IDP_GROUP", "IDP_DOUBLE"|
| IDProperty.subtype | StringValue| "IDP_INT", "IDP_FLOAT", "IDP_GROUP", "IDP_DOUBLE" |
| IDProperty.value | StringValue | When type is IDP_STRING |
| IDProperty.value | IntValue | When type is IDP_INT |
| IDProperty.value | DoubleValue | When type is IDP_FLOAT/IDP_DOUBLE |
| IDProperty.value | ArrayValue | When type is IDP_GROUP. Recursively uses the same structure as described in this section. |
| IDProperty.value | ArrayValue | When type is IDP_ARRAY. Each element holds a single element as described in this section. |

NOTE: IDP_ID and IDP_IDARRAY aren't supported. The entry will not be added.

Example
```
[
  {
    "name": "MyIntValue,
    "type": "IDP_INT",
    "value": 6,
  },
  {
    "name": "myComplexArray",
    "type": "IDP_ARRAY",
    "subtype": "IDP_GROUP",
    "value": [
        [
          {
            "name": ..
            ....
          }
        ]
    ]
  }
]
```

## Considered alternatives ##

- Add conversion functions inside `asset_indexer`; makes generic code part of a specific solution.
- Add conversion functions inside `BLI_serialize`; would add data transformation responsibilities inside a unit that is currently only responsible for formatting.
- Use direct mapping between IDP properties and Values; leads to missing information and edge cases (empty primitive arrays) that could not be de-serialized.

Reviewed By: Severin, mont29, HooglyBoogly

Maniphest Tasks: T92306

Differential Revision: https://developer.blender.org/D12990

1 files changed, 448 insertions, 0 deletions

diff --git a/source/blender/blenkernel/intern/idprop_serialize_test.cc b/source/blender/blenkernel/intern/idprop_serialize_test.cc
new file mode 100644
index 00000000000..eeee3fc2aea
--- /dev/null
+++ b/source/blender/blenkernel/intern/idprop_serialize_test.cc
@@ -0,0 +1,448 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 by Blender Foundation.
+ */
+
+#include "testing/testing.h"
+
+#include "DNA_ID.h"
+
+#include "BKE_idprop.hh"
+
+namespace blender::bke::idprop::tests {
+
+using namespace blender::io::serialize;
+
+static void check_container_value(ArrayValue *value)
+{
+  ASSERT_NE(value, nullptr);
+  ASSERT_EQ(value->type(), eValueType::Array);
+  const ArrayValue::Items elements = value->elements();
+  EXPECT_FALSE(elements.is_empty());
+  EXPECT_EQ(elements.size(), 1);
+
+  const ArrayValue::Item &item = value->elements()[0];
+  ASSERT_EQ(item->type(), eValueType::Dictionary);
+}
+
+static void check_object_attribute(const DictionaryValue::Lookup &lookup,
+                                   const std::string expected_key,
+                                   const std::string expected_value)
+{
+  EXPECT_TRUE(lookup.contains(expected_key));
+  const std::shared_ptr<Value> &element = *lookup.lookup_ptr(expected_key);
+  ASSERT_EQ(element->type(), eValueType::String);
+  EXPECT_EQ(element->as_string_value()->value(), expected_value);
+}
+
+static void check_object_attribute(const DictionaryValue::Lookup &lookup,
+                                   const std::string expected_key,
+                                   const int32_t expected_value)
+{
+  EXPECT_TRUE(lookup.contains(expected_key));
+  const std::shared_ptr<Value> &element = *lookup.lookup_ptr(expected_key);
+  ASSERT_EQ(element->type(), eValueType::Int);
+  EXPECT_EQ(element->as_int_value()->value(), expected_value);
+}
+
+static void check_object_attribute(const DictionaryValue::Lookup &lookup,
+                                   const std::string expected_key,
+                                   const float expected_value)
+{
+  EXPECT_TRUE(lookup.contains(expected_key));
+  const std::shared_ptr<Value> &element = *lookup.lookup_ptr(expected_key);
+  ASSERT_EQ(element->type(), eValueType::Double);
+  EXPECT_EQ(element->as_double_value()->value(), expected_value);
+}
+
+static void check_object_attribute(const DictionaryValue::Lookup &lookup,
+                                   const std::string expected_key,
+                                   const double expected_value)
+{
+  EXPECT_TRUE(lookup.contains(expected_key));
+  const std::shared_ptr<Value> &element = *lookup.lookup_ptr(expected_key);
+  ASSERT_EQ(element->type(), eValueType::Double);
+  EXPECT_EQ(element->as_double_value()->value(), expected_value);
+}
+
+static void test_string_to_value(const StringRefNull prop_name, const StringRefNull prop_content)
+{
+  std::unique_ptr<IDProperty, IDPropertyDeleter> property = create(prop_name, prop_content);
+
+  std::unique_ptr<ArrayValue> value = convert_to_serialize_values(property.get());
+  check_container_value(value.get());
+  const ArrayValue::Item &item = value->elements()[0];
+  const DictionaryValue *object = item->as_dictionary_value();
+  const DictionaryValue::Lookup lookup = object->create_lookup();
+
+  EXPECT_EQ(lookup.size(), 3);
+  check_object_attribute(lookup, "name", prop_name);
+  check_object_attribute(lookup, "type", "IDP_STRING");
+  check_object_attribute(lookup, "value", prop_content);
+}
+
+TEST(idprop, convert_idp_string_to_value)
+{
+  test_string_to_value("mykey", "mycontent");
+}
+
+static void test_int_to_value(const StringRefNull prop_name, int32_t prop_content)
+{
+  std::unique_ptr<IDProperty, IDPropertyDeleter> property = create(prop_name, prop_content);
+
+  std::unique_ptr<ArrayValue> value = convert_to_serialize_values(property.get());
+  check_container_value(value.get());
+  const ArrayValue::Item &item = value->elements()[0];
+  const DictionaryValue *object = item->as_dictionary_value();
+  const DictionaryValue::Lookup lookup = object->create_lookup();
+
+  EXPECT_EQ(lookup.size(), 3);
+  check_object_attribute(lookup, "name", prop_name);
+  check_object_attribute(lookup, "type", "IDP_INT");
+  check_object_attribute(lookup, "value", prop_content);
+}
+
+TEST(idprop, convert_idp_int_to_value)
+{
+  test_int_to_value("mykey", 0);
+}
+
+static void test_float_to_value(const StringRefNull prop_name, float prop_content)
+{
+  std::unique_ptr<IDProperty, IDPropertyDeleter> property = create(prop_name, prop_content);
+
+  std::unique_ptr<ArrayValue> value = convert_to_serialize_values(property.get());
+  check_container_value(value.get());
+  const ArrayValue::Item &item = value->elements()[0];
+  const DictionaryValue *object = item->as_dictionary_value();
+  const DictionaryValue::Lookup lookup = object->create_lookup();
+
+  EXPECT_EQ(lookup.size(), 3);
+  check_object_attribute(lookup, "name", prop_name);
+  check_object_attribute(lookup, "type", "IDP_FLOAT");
+  check_object_attribute(lookup, "value", prop_content);
+}
+
+TEST(idprop, convert_idp_float_to_value)
+{
+  test_float_to_value("mykey", 0.2f);
+}
+
+static void test_double_to_value(const StringRefNull prop_name, double prop_content)
+{
+  std::unique_ptr<IDProperty, IDPropertyDeleter> property = create(prop_name, prop_content);
+
+  std::unique_ptr<ArrayValue> value = convert_to_serialize_values(property.get());
+  check_container_value(value.get());
+  const ArrayValue::Item &item = value->elements()[0];
+  const DictionaryValue *object = item->as_dictionary_value();
+  const DictionaryValue::Lookup lookup = object->create_lookup();
+
+  EXPECT_EQ(lookup.size(), 3);
+  check_object_attribute(lookup, "name", prop_name);
+  check_object_attribute(lookup, "type", "IDP_DOUBLE");
+  check_object_attribute(lookup, "value", prop_content);
+}
+
+TEST(idprop, convert_idp_double_to_value)
+{
+  test_double_to_value("mykey", 0.2);
+}
+
+template<typename PrimitiveType, typename ValueType>
+static void test_array_to_value(const StringRefNull prop_name, Vector<PrimitiveType> prop_content)
+{
+  std::unique_ptr<IDProperty, IDPropertyDeleter> property = create(prop_name, prop_content);
+  std::unique_ptr<ArrayValue> value = convert_to_serialize_values(property.get());
+
+  check_container_value(value.get());
+  const ArrayValue::Item &item = value->elements()[0];
+  const DictionaryValue *object = item->as_dictionary_value();
+  const DictionaryValue::Lookup lookup = object->create_lookup();
+
+  EXPECT_EQ(lookup.size(), 4);
+  check_object_attribute(lookup, "name", prop_name);
+  check_object_attribute(lookup, "type", "IDP_ARRAY");
+
+  const std::shared_ptr<Value> &element = *lookup.lookup_ptr("value");
+  const ArrayValue *subvalues = element->as_array_value();
+  ASSERT_NE(subvalues, nullptr);
+  const ArrayValue::Items &subitems = subvalues->elements();
+  ASSERT_EQ(subitems.size(), prop_content.size());
+
+  for (size_t i = 0; i < prop_content.size(); i++) {
+    EXPECT_EQ(static_cast<ValueType *>(subitems[i].get())->value(), prop_content[i]);
+  }
+}
+
+TEST(idprop, convert_idp_int_array_to_value)
+{
+  test_array_to_value<int32_t, IntValue>("my_integer_array",
+                                         {-16, -8, -4, -2, -1, 0, 1, 2, 4, 8, 16});
+}
+
+TEST(idprop, convert_idp_float_array_to_value)
+{
+  test_array_to_value<float, DoubleValue>(
+      "my_float_array", {-16.8f, -8.4f, -4.2f, -2.1f, -1.0f, 0.0f, 1.0f, 2.1f, 4.2f, 8.4f, 16.8f});
+}
+
+TEST(idprop, convert_idp_double_array_to_value)
+{
+  test_array_to_value<double, DoubleValue>(
+      "my_double_array", {-16.8, -8.4, -4.2, -2.1, -1.0, 0.0, 1.0, 2.1, 4.2, 8.4, 16.8});
+}
+
+static std::unique_ptr<Value> parse_json(StringRef input)
+{
+  std::stringstream is(input);
+  JsonFormatter json;
+  std::unique_ptr<Value> value = json.deserialize(is);
+  return value;
+}
+
+static std::string to_json(const Value &value)
+{
+  std::stringstream out;
+  JsonFormatter json;
+  json.serialize(out, value);
+  return out.str();
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        StringRef expected_value)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_STRING);
+  EXPECT_EQ(id_property->name, expected_name);
+  EXPECT_EQ(IDP_String(id_property), expected_value);
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        int32_t expected_value)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_INT);
+  EXPECT_EQ(id_property->name, expected_name);
+  EXPECT_EQ(IDP_Int(id_property), expected_value);
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        float expected_value)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_FLOAT);
+  EXPECT_EQ(id_property->name, expected_name);
+  EXPECT_EQ(IDP_Float(id_property), expected_value);
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        double expected_value)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_DOUBLE);
+  EXPECT_EQ(id_property->name, expected_name);
+  EXPECT_EQ(IDP_Double(id_property), expected_value);
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        const Vector<int32_t> &values)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_ARRAY);
+  EXPECT_EQ(id_property->subtype, IDP_INT);
+  EXPECT_EQ(id_property->len, values.size());
+  EXPECT_EQ(id_property->name, expected_name);
+  int32_t *idprop_values = static_cast<int32_t *>(IDP_Array(id_property));
+  for (int i = 0; i < values.size(); i++) {
+    EXPECT_EQ(idprop_values[i], values[i]);
+  }
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        const Vector<float> &values)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_ARRAY);
+  EXPECT_EQ(id_property->subtype, IDP_FLOAT);
+  EXPECT_EQ(id_property->len, values.size());
+  EXPECT_EQ(id_property->name, expected_name);
+  float *idprop_values = static_cast<float *>(IDP_Array(id_property));
+  for (int i = 0; i < values.size(); i++) {
+    EXPECT_EQ(idprop_values[i], values[i]);
+  }
+}
+
+static void test_idprop(const IDProperty *id_property,
+                        StringRef expected_name,
+                        const Vector<double> &values)
+{
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_ARRAY);
+  EXPECT_EQ(id_property->subtype, IDP_DOUBLE);
+  EXPECT_EQ(id_property->len, values.size());
+  EXPECT_EQ(id_property->name, expected_name);
+  double *idprop_values = static_cast<double *>(IDP_Array(id_property));
+  for (int i = 0; i < values.size(); i++) {
+    EXPECT_EQ(idprop_values[i], values[i]);
+  }
+}
+
+template<typename Type>
+static void test_convert_idprop_from_value(StringRef input,
+                                           StringRef expected_name,
+                                           Type expected_value)
+{
+  std::unique_ptr<Value> value = parse_json(input);
+  IDProperty *id_property = convert_from_serialize_value(*value);
+  test_idprop(id_property, expected_name, expected_value);
+  IDP_FreeProperty(id_property);
+}
+
+TEST(idprop, convert_idp_string_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyStringName","type":"IDP_STRING","value":"MyString"}])",
+      "MyStringName",
+      "MyString");
+}
+
+TEST(idprop, convert_idp_int_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyIntegerName","type":"IDP_INT","value":42}])", "MyIntegerName", 42);
+}
+
+TEST(idprop, convert_idp_float_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyFloatName","type":"IDP_FLOAT","value":42.24}])", "MyFloatName", 42.24f);
+}
+
+TEST(idprop, convert_idp_double_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyDoubleName","type":"IDP_DOUBLE","value":42.24}])", "MyDoubleName", 42.24);
+}
+
+TEST(idprop, convert_idp_array_int_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyArrayName","type":"IDP_ARRAY","subtype":"IDP_INT","value":[42, 24, 35]}])",
+      "MyArrayName",
+      Vector<int32_t>{42, 24, 35});
+}
+
+TEST(idprop, convert_idp_array_float_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyArrayName","type":"IDP_ARRAY","subtype":"IDP_FLOAT","value":[42.0, 24.4, 35.2]}])",
+      "MyArrayName",
+      Vector<float>{42.0f, 24.4f, 35.2f});
+}
+
+TEST(idprop, convert_idp_array_double_from_value)
+{
+  test_convert_idprop_from_value(
+      R"([{"name":"MyArrayName","type":"IDP_ARRAY","subtype":"IDP_DOUBLE","value":[42.43,24.5,35.8]}])",
+      "MyArrayName",
+      Vector<double>{42.43, 24.5, 35.8});
+}
+
+TEST(idprop, convert_idp_multiple_from_value)
+{
+  static const std::string input_json =
+      R"([{"name":"MyIntegerName","type":"IDP_INT","value":42},{"name":"MyStringName","type":"IDP_STRING","value":"MyString"},{"name":"MyFloatName","type":"IDP_FLOAT","value":42.24},{"name":"MyDoubleName","type":"IDP_DOUBLE","value":42.24}])";
+  std::unique_ptr<Value> value = parse_json(input_json);
+
+  IDProperty *id_property = convert_from_serialize_value(*value);
+  IDProperty *id_property_1 = id_property;
+  ASSERT_NE(id_property_1, nullptr);
+  IDProperty *id_property_2 = id_property_1->next;
+  ASSERT_NE(id_property_2, nullptr);
+  IDProperty *id_property_3 = id_property_2->next;
+  ASSERT_NE(id_property_3, nullptr);
+  IDProperty *id_property_4 = id_property_3->next;
+  ASSERT_NE(id_property_4, nullptr);
+
+  EXPECT_EQ(id_property_1->prev, nullptr);
+  EXPECT_EQ(id_property_2->prev, id_property_1);
+  EXPECT_EQ(id_property_3->prev, id_property_2);
+  EXPECT_EQ(id_property_4->prev, id_property_3);
+  EXPECT_EQ(id_property_4->next, nullptr);
+
+  test_idprop(id_property_1, "MyIntegerName", 42);
+  test_idprop(id_property_2, "MyStringName", "MyString");
+  test_idprop(id_property_3, "MyFloatName", 42.24f);
+  test_idprop(id_property_4, "MyDoubleName", 42.24);
+
+  IDP_FreeProperty(id_property_1);
+  IDP_FreeProperty(id_property_2);
+  IDP_FreeProperty(id_property_3);
+  IDP_FreeProperty(id_property_4);
+}
+
+TEST(idprop, convert_idp_multiple_roundtrip)
+{
+  static const std::string input_json =
+      R"([{"name":"MyIntegerName","type":"IDP_INT","value":42},{"name":"MyStringName","type":"IDP_STRING","value":"MyString"},{"name":"MyFloatName","type":"IDP_FLOAT","value":42.2400016784668},{"name":"MyDoubleName","type":"IDP_DOUBLE","value":42.24}])";
+  std::unique_ptr<Value> value = parse_json(input_json);
+
+  IDProperty *id_property = convert_from_serialize_value(*value);
+  IDProperty *id_property_1 = id_property;
+  ASSERT_NE(id_property_1, nullptr);
+  IDProperty *id_property_2 = id_property_1->next;
+  ASSERT_NE(id_property_2, nullptr);
+  IDProperty *id_property_3 = id_property_2->next;
+  ASSERT_NE(id_property_3, nullptr);
+  IDProperty *id_property_4 = id_property_3->next;
+  ASSERT_NE(id_property_4, nullptr);
+
+  std::unique_ptr<Value> value_from_id_properties = convert_to_serialize_values(id_property);
+  std::string output_json = to_json(*value_from_id_properties);
+  EXPECT_EQ(input_json, output_json);
+
+  IDP_FreeProperty(id_property_1);
+  IDP_FreeProperty(id_property_2);
+  IDP_FreeProperty(id_property_3);
+  IDP_FreeProperty(id_property_4);
+}
+
+TEST(idprop, convert_idp_group_from_value)
+{
+  static const std::string input_json =
+      R"([{"name":"AssetMetaData.properties","type":"IDP_GROUP","value":[{"name":"dimensions","type":"IDP_ARRAY","subtype":"IDP_FLOAT","value":[2.0,2.0,2.0]}]}])";
+  std::unique_ptr<Value> value = parse_json(input_json);
+
+  IDProperty *id_property = convert_from_serialize_value(*value);
+  ASSERT_NE(id_property, nullptr);
+  EXPECT_EQ(id_property->type, IDP_GROUP);
+  EXPECT_EQ(BLI_listbase_count(&id_property->data.group), 1);
+
+  test_idprop(static_cast<IDProperty *>(id_property->data.group.first),
+              "dimensions",
+              Vector<float>{2.0f, 2.0f, 2.0f});
+
+  IDP_FreeProperty(id_property);
+}
+
+}  // namespace blender::bke::idprop::tests