From 213cd39b6db387bd88f12589fd50ff0e6563cf56 Mon Sep 17 00:00:00 2001 From: Aras Pranckevicius Date: Sun, 17 Apr 2022 22:07:43 +0300 Subject: OBJ: further optimize, cleanup and harden the new C++ importer Continued improvements to the new C++ based OBJ importer. Performance: about 2x faster. - Rungholt.obj (several meshes, 263MB file): Windows 12.7s -> 5.9s, Mac 7.7s -> 3.1s. - Blender 3.0 splash (24k meshes, 2.4GB file): Windows 97.3s -> 53.6s, Mac 137.3s -> 80.0s. - "Windows" is VS2022, AMD Ryzen 5950X (32 threads), "Mac" is Xcode/clang 13, M1Max (10 threads). - Slightly reduced memory usage during import as well. The performance gains are a combination of several things: - Replacing `std::stof` / `std::stoi` with C++17 `from_chars`. - Stop reading input file char-by-char using `std::getline`, and instead read in 64kb chunks, and parse from there (taking care of possibly handling lines split mid-way due to chunk boundaries). - Removing abstractions for splitting a line by some char, - Avoid tiny memory allocations: instead of storing a vector of polygon corners in each face, store all the corners in one big array, and per-face only store indices "where do corners start, and how many". Likewise, don't store full string names of material/group names for each face; only store indices into overall material/group names arrays. - Stop always doing mesh validation, which is slow. Do it just like the Alembic importer does: only do validation if found some invalid faces during import, or if requested by the user via an import setting checkbox (which defaults to off). - Stop doing "collection sync" for each object being added; instead do the collection sync right after creating all the objects. Cleanup / Robustness: This reworking of parser (see "removing abstractions" point above) means that all the functions that were in `parser_string_utils` file are gone, and replaced with different set of functions. However they are not OBJ specific, so as pointed out during review of the previous differential, they are now in `source/blender/io/common` library. Added gtest coverage for said functions as well; something that was only indirectly covered by obj tests previously. Rework of some bits of parsing made the parser actually better able to deal with invalid syntax. E.g. previously, if a face corner were a `/123` string, it would have incorrectly treated that as a vertex index (since it would get "hey that's one number" after splitting a string by a slash), instead of properly marking it as invalid syntax. Added gtest coverage for .mtl parsing; something that was not covered by any tests at all previously. Reviewed By: Howard Trickey Differential Revision: https://developer.blender.org/D14586 --- source/blender/editors/io/io_obj.c | 13 +- source/blender/io/common/CMakeLists.txt | 5 + source/blender/io/common/IO_string_utils.hh | 69 ++ source/blender/io/common/intern/string_utils.cc | 99 +++ .../blender/io/common/intern/string_utils_test.cc | 118 +++ source/blender/io/wavefront_obj/CMakeLists.txt | 5 +- source/blender/io/wavefront_obj/IO_wavefront_obj.h | 1 + .../importer/obj_import_file_reader.cc | 793 +++++++++++---------- .../importer/obj_import_file_reader.hh | 114 +-- .../io/wavefront_obj/importer/obj_import_mesh.cc | 95 +-- .../io/wavefront_obj/importer/obj_import_mesh.hh | 7 +- .../io/wavefront_obj/importer/obj_import_mtl.cc | 10 +- .../io/wavefront_obj/importer/obj_import_mtl.hh | 25 - .../wavefront_obj/importer/obj_import_objects.hh | 25 +- .../io/wavefront_obj/importer/obj_importer.cc | 37 +- .../io/wavefront_obj/importer/obj_importer.hh | 3 +- .../wavefront_obj/importer/parser_string_utils.cc | 174 ----- .../wavefront_obj/importer/parser_string_utils.hh | 54 -- .../io/wavefront_obj/tests/obj_importer_tests.cc | 3 +- .../io/wavefront_obj/tests/obj_mtl_parser_tests.cc | 172 +++++ 20 files changed, 1020 insertions(+), 802 deletions(-) create mode 100644 source/blender/io/common/IO_string_utils.hh create mode 100644 source/blender/io/common/intern/string_utils.cc create mode 100644 source/blender/io/common/intern/string_utils_test.cc delete mode 100644 source/blender/io/wavefront_obj/importer/parser_string_utils.cc delete mode 100644 source/blender/io/wavefront_obj/importer/parser_string_utils.hh create mode 100644 source/blender/io/wavefront_obj/tests/obj_mtl_parser_tests.cc (limited to 'source') diff --git a/source/blender/editors/io/io_obj.c b/source/blender/editors/io/io_obj.c index 97f1e08fdff..beed4abd52b 100644 --- a/source/blender/editors/io/io_obj.c +++ b/source/blender/editors/io/io_obj.c @@ -378,6 +378,7 @@ static int wm_obj_import_exec(bContext *C, wmOperator *op) import_params.clamp_size = RNA_float_get(op->ptr, "clamp_size"); import_params.forward_axis = RNA_enum_get(op->ptr, "forward_axis"); import_params.up_axis = RNA_enum_get(op->ptr, "up_axis"); + import_params.validate_meshes = RNA_boolean_get(op->ptr, "validate_meshes"); OBJ_import(C, &import_params); @@ -388,8 +389,8 @@ static void ui_obj_import_settings(uiLayout *layout, PointerRNA *imfptr) { uiLayoutSetPropSep(layout, true); uiLayoutSetPropDecorate(layout, false); - uiLayout *box = uiLayoutBox(layout); + uiLayout *box = uiLayoutBox(layout); uiItemL(box, IFACE_("Transform"), ICON_OBJECT_DATA); uiLayout *col = uiLayoutColumn(box, false); uiLayout *sub = uiLayoutColumn(col, false); @@ -397,6 +398,11 @@ static void ui_obj_import_settings(uiLayout *layout, PointerRNA *imfptr) sub = uiLayoutColumn(col, false); uiItemR(sub, imfptr, "forward_axis", 0, IFACE_("Axis Forward"), ICON_NONE); uiItemR(sub, imfptr, "up_axis", 0, IFACE_("Up"), ICON_NONE); + + box = uiLayoutBox(layout); + uiItemL(box, IFACE_("Options"), ICON_EXPORT); + col = uiLayoutColumn(box, false); + uiItemR(col, imfptr, "validate_meshes", 0, NULL, ICON_NONE); } static void wm_obj_import_draw(bContext *C, wmOperator *op) @@ -442,4 +448,9 @@ void WM_OT_obj_import(struct wmOperatorType *ot) "Forward Axis", ""); RNA_def_enum(ot->srna, "up_axis", io_obj_transform_axis_up, OBJ_AXIS_Y_UP, "Up Axis", ""); + RNA_def_boolean(ot->srna, + "validate_meshes", + false, + "Validate Meshes", + "Check imported mesh objects for invalid data (slow)"); } diff --git a/source/blender/io/common/CMakeLists.txt b/source/blender/io/common/CMakeLists.txt index 02bd5b2b81f..b1add38bf01 100644 --- a/source/blender/io/common/CMakeLists.txt +++ b/source/blender/io/common/CMakeLists.txt @@ -7,6 +7,8 @@ set(INC ../../blenlib ../../depsgraph ../../makesdna + ../../../../intern/guardedalloc + ../../../../extern/fast_float ) set(INC_SYS @@ -17,9 +19,11 @@ set(SRC intern/dupli_parent_finder.cc intern/dupli_persistent_id.cc intern/object_identifier.cc + intern/string_utils.cc IO_abstract_hierarchy_iterator.h IO_dupli_persistent_id.hh + IO_string_utils.hh IO_types.h intern/dupli_parent_finder.hh ) @@ -38,6 +42,7 @@ if(WITH_GTESTS) intern/abstract_hierarchy_iterator_test.cc intern/hierarchy_context_order_test.cc intern/object_identifier_test.cc + intern/string_utils_test.cc ) set(TEST_INC ../../blenloader diff --git a/source/blender/io/common/IO_string_utils.hh b/source/blender/io/common/IO_string_utils.hh new file mode 100644 index 00000000000..25f1f01c6ed --- /dev/null +++ b/source/blender/io/common/IO_string_utils.hh @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#pragma once + +#include "BLI_string_ref.hh" + +/* + * Various text parsing utilities commonly used by text-based input formats. + */ + +namespace blender::io { + +/** + * Fetches next line from an input string buffer. + * + * The returned line will not have '\n' characters at the end; + * the `buffer` is modified to contain remaining text without + * the input line. + * + * Note that backslash (\) character is treated as a line + * continuation, similar to OBJ file format or a C preprocessor. + */ +StringRef read_next_line(StringRef &buffer); + +/** + * Drop leading white-space from a StringRef. + * Note that backslash character is considered white-space. + */ +StringRef drop_whitespace(StringRef str); + +/** + * Drop leading non-white-space from a StringRef. + * Note that backslash character is considered white-space. + */ +StringRef drop_non_whitespace(StringRef str); + +/** + * Parse an integer from an input string. + * The parsed result is stored in `dst`. The function skips + * leading white-space unless `skip_space=false`. If the + * number can't be parsed (invalid syntax, out of range), + * `fallback` value is stored instead. + * + * Returns the remainder of the input string after parsing. + */ +StringRef parse_int(StringRef str, int fallback, int &dst, bool skip_space = true); + +/** + * Parse a float from an input string. + * The parsed result is stored in `dst`. The function skips + * leading white-space unless `skip_space=false`. If the + * number can't be parsed (invalid syntax, out of range), + * `fallback` value is stored instead. + * + * Returns the remainder of the input string after parsing. + */ +StringRef parse_float(StringRef str, float fallback, float &dst, bool skip_space = true); + +/** + * Parse a number of white-space separated floats from an input string. + * The parsed `count` numbers are stored in `dst`. If a + * number can't be parsed (invalid syntax, out of range), + * `fallback` value is stored instead. + * + * Returns the remainder of the input string after parsing. + */ +StringRef parse_floats(StringRef str, float fallback, float *dst, int count); + +} // namespace blender::io diff --git a/source/blender/io/common/intern/string_utils.cc b/source/blender/io/common/intern/string_utils.cc new file mode 100644 index 00000000000..01107b0866e --- /dev/null +++ b/source/blender/io/common/intern/string_utils.cc @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include "IO_string_utils.hh" + +/* Note: we could use C++17 from_chars to parse + * floats, but even if some compilers claim full support, + * their standard libraries are not quite there yet. + * LLVM/libc++ only has a float parser since LLVM 14, + * and gcc/libstdc++ since 11.1. So until at least these are + * the mininum spec, use an external library. */ +#include "fast_float.h" +#include + +namespace blender::io { + +StringRef read_next_line(StringRef &buffer) +{ + const char *start = buffer.begin(); + const char *end = buffer.end(); + size_t len = 0; + char prev = 0; + const char *ptr = start; + while (ptr < end) { + char c = *ptr++; + if (c == '\n' && prev != '\\') { + break; + } + prev = c; + ++len; + } + + buffer = StringRef(ptr, end); + return StringRef(start, len); +} + +static bool is_whitespace(char c) +{ + return c <= ' ' || c == '\\'; +} + +StringRef drop_whitespace(StringRef str) +{ + while (!str.is_empty() && is_whitespace(str[0])) { + str = str.drop_prefix(1); + } + return str; +} + +StringRef drop_non_whitespace(StringRef str) +{ + while (!str.is_empty() && !is_whitespace(str[0])) { + str = str.drop_prefix(1); + } + return str; +} + +static StringRef drop_plus(StringRef str) +{ + if (!str.is_empty() && str[0] == '+') { + str = str.drop_prefix(1); + } + return str; +} + +StringRef parse_float(StringRef str, float fallback, float &dst, bool skip_space) +{ + if (skip_space) { + str = drop_whitespace(str); + } + str = drop_plus(str); + fast_float::from_chars_result res = fast_float::from_chars(str.begin(), str.end(), dst); + if (res.ec == std::errc::invalid_argument || res.ec == std::errc::result_out_of_range) { + dst = fallback; + } + return StringRef(res.ptr, str.end()); +} + +StringRef parse_floats(StringRef str, float fallback, float *dst, int count) +{ + for (int i = 0; i < count; ++i) { + str = parse_float(str, fallback, dst[i]); + } + return str; +} + +StringRef parse_int(StringRef str, int fallback, int &dst, bool skip_space) +{ + if (skip_space) { + str = drop_whitespace(str); + } + str = drop_plus(str); + std::from_chars_result res = std::from_chars(str.begin(), str.end(), dst); + if (res.ec == std::errc::invalid_argument || res.ec == std::errc::result_out_of_range) { + dst = fallback; + } + return StringRef(res.ptr, str.end()); +} + +} // namespace blender::io diff --git a/source/blender/io/common/intern/string_utils_test.cc b/source/blender/io/common/intern/string_utils_test.cc new file mode 100644 index 00000000000..a78bd7ab8a3 --- /dev/null +++ b/source/blender/io/common/intern/string_utils_test.cc @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "IO_string_utils.hh" + +#include "testing/testing.h" + +namespace blender::io { + +#define EXPECT_STRREF_EQ(str1, str2) EXPECT_STREQ(str1, std::string(str2).c_str()) + +TEST(string_utils, read_next_line) +{ + std::string str = "abc\n \n\nline with \\\ncontinuation\nCRLF ending:\r\na"; + StringRef s = str; + EXPECT_STRREF_EQ("abc", read_next_line(s)); + EXPECT_STRREF_EQ(" ", read_next_line(s)); + EXPECT_STRREF_EQ("", read_next_line(s)); + EXPECT_STRREF_EQ("line with \\\ncontinuation", read_next_line(s)); + EXPECT_STRREF_EQ("CRLF ending:\r", read_next_line(s)); + EXPECT_STRREF_EQ("a", read_next_line(s)); + EXPECT_TRUE(s.is_empty()); +} + +TEST(string_utils, drop_whitespace) +{ + /* Empty */ + EXPECT_STRREF_EQ("", drop_whitespace("")); + /* Only whitespace */ + EXPECT_STRREF_EQ("", drop_whitespace(" ")); + EXPECT_STRREF_EQ("", drop_whitespace(" ")); + EXPECT_STRREF_EQ("", drop_whitespace(" \t\n\r ")); + /* Drops leading whitespace */ + EXPECT_STRREF_EQ("a", drop_whitespace(" a")); + EXPECT_STRREF_EQ("a b", drop_whitespace(" a b")); + EXPECT_STRREF_EQ("a b ", drop_whitespace(" a b ")); + /* No leading whitespace */ + EXPECT_STRREF_EQ("c", drop_whitespace("c")); + /* Case with backslash, should be treated as whitespace */ + EXPECT_STRREF_EQ("d", drop_whitespace(" \\ d")); +} + +TEST(string_utils, parse_int_valid) +{ + std::string str = "1 -10 \t 1234 1234567890 +7 123a"; + StringRef s = str; + int val; + s = parse_int(s, 0, val); + EXPECT_EQ(1, val); + s = parse_int(s, 0, val); + EXPECT_EQ(-10, val); + s = parse_int(s, 0, val); + EXPECT_EQ(1234, val); + s = parse_int(s, 0, val); + EXPECT_EQ(1234567890, val); + s = parse_int(s, 0, val); + EXPECT_EQ(7, val); + s = parse_int(s, 0, val); + EXPECT_EQ(123, val); + EXPECT_STRREF_EQ("a", s); +} + +TEST(string_utils, parse_int_invalid) +{ + int val; + /* Invalid syntax */ + EXPECT_STRREF_EQ("--123", parse_int("--123", -1, val)); + EXPECT_EQ(val, -1); + EXPECT_STRREF_EQ("foobar", parse_int("foobar", -2, val)); + EXPECT_EQ(val, -2); + /* Out of integer range */ + EXPECT_STRREF_EQ(" a", parse_int("1234567890123 a", -3, val)); + EXPECT_EQ(val, -3); + /* Has leading white-space when we don't expect it */ + EXPECT_STRREF_EQ(" 1", parse_int(" 1", -4, val, false)); + EXPECT_EQ(val, -4); +} + +TEST(string_utils, parse_float_valid) +{ + std::string str = "1 -10 123.5 -17.125 0.1 1e6 50.0e-1"; + StringRef s = str; + float val; + s = parse_float(s, 0, val); + EXPECT_EQ(1.0f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(-10.0f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(123.5f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(-17.125f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(0.1f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(1.0e6f, val); + s = parse_float(s, 0, val); + EXPECT_EQ(5.0f, val); + EXPECT_TRUE(s.is_empty()); +} + +TEST(string_utils, parse_float_invalid) +{ + float val; + /* Invalid syntax */ + EXPECT_STRREF_EQ("_0", parse_float("_0", -1.0f, val)); + EXPECT_EQ(val, -1.0f); + EXPECT_STRREF_EQ("..5", parse_float("..5", -2.0f, val)); + EXPECT_EQ(val, -2.0f); + /* Out of float range. Current float parser (fast_float) + * clamps out of range numbers to +/- infinity, so this + * one gets a +inf instead of fallback -3.0. */ + EXPECT_STRREF_EQ(" a", parse_float("9.0e500 a", -3.0f, val)); + EXPECT_EQ(val, std::numeric_limits::infinity()); + /* Has leading white-space when we don't expect it */ + EXPECT_STRREF_EQ(" 1", parse_float(" 1", -4.0f, val, false)); + EXPECT_EQ(val, -4.0f); +} + +} // namespace blender::io diff --git a/source/blender/io/wavefront_obj/CMakeLists.txt b/source/blender/io/wavefront_obj/CMakeLists.txt index 9cdd96ee7be..67cec000778 100644 --- a/source/blender/io/wavefront_obj/CMakeLists.txt +++ b/source/blender/io/wavefront_obj/CMakeLists.txt @@ -4,6 +4,7 @@ set(INC . ./exporter ./importer + ../common ../../blenkernel ../../blenlib ../../bmesh @@ -35,7 +36,6 @@ set(SRC importer/obj_import_mtl.cc importer/obj_import_nurbs.cc importer/obj_importer.cc - importer/parser_string_utils.cc IO_wavefront_obj.h exporter/obj_export_file_writer.hh @@ -51,11 +51,11 @@ set(SRC importer/obj_import_nurbs.hh importer/obj_import_objects.hh importer/obj_importer.hh - importer/parser_string_utils.hh ) set(LIB bf_blenkernel + bf_io_common ) if(WITH_TBB) @@ -70,6 +70,7 @@ if(WITH_GTESTS) set(TEST_SRC tests/obj_exporter_tests.cc tests/obj_importer_tests.cc + tests/obj_mtl_parser_tests.cc tests/obj_exporter_tests.hh ) diff --git a/source/blender/io/wavefront_obj/IO_wavefront_obj.h b/source/blender/io/wavefront_obj/IO_wavefront_obj.h index b06ccbf8702..8b71ec750c0 100644 --- a/source/blender/io/wavefront_obj/IO_wavefront_obj.h +++ b/source/blender/io/wavefront_obj/IO_wavefront_obj.h @@ -84,6 +84,7 @@ struct OBJImportParams { float clamp_size; eTransformAxisForward forward_axis; eTransformAxisUp up_axis; + bool validate_meshes; }; /** diff --git a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc index d3d4e1ba860..184810b9802 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc +++ b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc @@ -8,7 +8,7 @@ #include "BLI_string_ref.hh" #include "BLI_vector.hh" -#include "parser_string_utils.hh" +#include "IO_string_utils.hh" #include "obj_import_file_reader.hh" @@ -34,7 +34,8 @@ static Geometry *create_geometry(Geometry *const prev_geometry, Geometry *g = r_all_geometries.last().get(); g->geom_type_ = new_type; g->geometry_name_ = name.is_empty() ? "New object" : name; - r_offset.set_index_offset(global_vertices.vertices.size()); + g->vertex_start_ = global_vertices.vertices.size(); + r_offset.set_index_offset(g->vertex_start_); return g; }; @@ -66,48 +67,40 @@ static Geometry *create_geometry(Geometry *const prev_geometry, } static void geom_add_vertex(Geometry *geom, - const StringRef rest_line, + const StringRef line, GlobalVertices &r_global_vertices) { - float3 curr_vert; - Vector str_vert_split; - split_by_char(rest_line, ' ', str_vert_split); - copy_string_to_float(str_vert_split, FLT_MAX, {curr_vert, 3}); - r_global_vertices.vertices.append(curr_vert); - geom->vertex_indices_.append(r_global_vertices.vertices.size() - 1); + float3 vert; + parse_floats(line, FLT_MAX, vert, 3); + r_global_vertices.vertices.append(vert); + geom->vertex_count_++; } static void geom_add_vertex_normal(Geometry *geom, - const StringRef rest_line, + const StringRef line, GlobalVertices &r_global_vertices) { - float3 curr_vert_normal; - Vector str_vert_normal_split; - split_by_char(rest_line, ' ', str_vert_normal_split); - copy_string_to_float(str_vert_normal_split, FLT_MAX, {curr_vert_normal, 3}); - r_global_vertices.vertex_normals.append(curr_vert_normal); + float3 normal; + parse_floats(line, FLT_MAX, normal, 3); + r_global_vertices.vertex_normals.append(normal); geom->has_vertex_normals_ = true; } -static void geom_add_uv_vertex(const StringRef rest_line, GlobalVertices &r_global_vertices) +static void geom_add_uv_vertex(const StringRef line, GlobalVertices &r_global_vertices) { - float2 curr_uv_vert; - Vector str_uv_vert_split; - split_by_char(rest_line, ' ', str_uv_vert_split); - copy_string_to_float(str_uv_vert_split, FLT_MAX, {curr_uv_vert, 2}); - r_global_vertices.uv_vertices.append(curr_uv_vert); + float2 uv; + parse_floats(line, FLT_MAX, uv, 2); + r_global_vertices.uv_vertices.append(uv); } static void geom_add_edge(Geometry *geom, - const StringRef rest_line, + StringRef line, const VertexIndexOffset &offsets, GlobalVertices &r_global_vertices) { - int edge_v1 = -1, edge_v2 = -1; - Vector str_edge_split; - split_by_char(rest_line, ' ', str_edge_split); - copy_string_to_int(str_edge_split[0], -1, edge_v1); - copy_string_to_int(str_edge_split[1], -1, edge_v2); + int edge_v1, edge_v2; + line = parse_int(line, -1, edge_v1); + line = parse_int(line, -1, edge_v2); /* Always keep stored indices non-negative and zero-based. */ edge_v1 += edge_v1 < 0 ? r_global_vertices.vertices.size() : -offsets.get_index_offset() - 1; edge_v2 += edge_v2 < 0 ? r_global_vertices.vertices.size() : -offsets.get_index_offset() - 1; @@ -116,78 +109,45 @@ static void geom_add_edge(Geometry *geom, } static void geom_add_polygon(Geometry *geom, - const StringRef rest_line, + StringRef line, const GlobalVertices &global_vertices, const VertexIndexOffset &offsets, - const StringRef state_material_name, - const StringRef state_object_group, - const bool state_shaded_smooth) + const int material_index, + const int group_index, + const bool shaded_smooth) { PolyElem curr_face; - curr_face.shaded_smooth = state_shaded_smooth; - if (!state_material_name.is_empty()) { - curr_face.material_name = state_material_name; - } - if (!state_object_group.is_empty()) { - curr_face.vertex_group = state_object_group; - /* Yes it repeats several times, but another if-check will not reduce steps either. */ + curr_face.shaded_smooth = shaded_smooth; + curr_face.material_index = material_index; + if (group_index >= 0) { + curr_face.vertex_group_index = group_index; geom->use_vertex_groups_ = true; } + const int orig_corners_size = geom->face_corners_.size(); + curr_face.start_index_ = orig_corners_size; + bool face_valid = true; - Vector str_corners_split; - split_by_char(rest_line, ' ', str_corners_split); - for (StringRef str_corner : str_corners_split) { + while (!line.is_empty() && face_valid) { PolyCorner corner; - const size_t n_slash = std::count(str_corner.begin(), str_corner.end(), '/'); bool got_uv = false, got_normal = false; - if (n_slash == 0) { - /* Case: "f v1 v2 v3". */ - copy_string_to_int(str_corner, INT32_MAX, corner.vert_index); - } - else if (n_slash == 1) { - /* Case: "f v1/vt1 v2/vt2 v3/vt3". */ - Vector vert_uv_split; - split_by_char(str_corner, '/', vert_uv_split); - if (vert_uv_split.size() != 1 && vert_uv_split.size() != 2) { - fprintf(stderr, "Invalid face syntax '%s', ignoring\n", std::string(str_corner).c_str()); - face_valid = false; + /* Parse vertex index. */ + line = parse_int(line, INT32_MAX, corner.vert_index, false); + face_valid &= corner.vert_index != INT32_MAX; + if (!line.is_empty() && line[0] == '/') { + /* Parse UV index. */ + line = line.drop_prefix(1); + if (!line.is_empty() && line[0] != '/') { + line = parse_int(line, INT32_MAX, corner.uv_vert_index, false); + got_uv = corner.uv_vert_index != INT32_MAX; } - else { - copy_string_to_int(vert_uv_split[0], INT32_MAX, corner.vert_index); - if (vert_uv_split.size() == 2) { - copy_string_to_int(vert_uv_split[1], INT32_MAX, corner.uv_vert_index); - got_uv = corner.uv_vert_index != INT32_MAX; - } + /* Parse normal index. */ + if (!line.is_empty() && line[0] == '/') { + line = line.drop_prefix(1); + line = parse_int(line, INT32_MAX, corner.vertex_normal_index, false); + got_normal = corner.uv_vert_index != INT32_MAX; } } - else if (n_slash == 2) { - /* Case: "f v1//vn1 v2//vn2 v3//vn3". */ - /* Case: "f v1/vt1/vn1 v2/vt2/vn2 v3/vt3/vn3". */ - Vector vert_uv_normal_split; - split_by_char(str_corner, '/', vert_uv_normal_split); - if (vert_uv_normal_split.size() != 2 && vert_uv_normal_split.size() != 3) { - fprintf(stderr, "Invalid face syntax '%s', ignoring\n", std::string(str_corner).c_str()); - face_valid = false; - } - else { - copy_string_to_int(vert_uv_normal_split[0], INT32_MAX, corner.vert_index); - if (vert_uv_normal_split.size() == 3) { - copy_string_to_int(vert_uv_normal_split[1], INT32_MAX, corner.uv_vert_index); - got_uv = corner.uv_vert_index != INT32_MAX; - copy_string_to_int(vert_uv_normal_split[2], INT32_MAX, corner.vertex_normal_index); - got_normal = corner.vertex_normal_index != INT32_MAX; - } - else { - copy_string_to_int(vert_uv_normal_split[1], INT32_MAX, corner.vertex_normal_index); - got_normal = corner.vertex_normal_index != INT32_MAX; - } - } - } - else { - fprintf(stderr, "Invalid face syntax '%s', ignoring\n", std::string(str_corner).c_str()); - face_valid = false; - } /* Always keep stored indices non-negative and zero-based. */ corner.vert_index += corner.vert_index < 0 ? global_vertices.vertices.size() : -offsets.get_index_offset() - 1; @@ -221,19 +181,28 @@ static void geom_add_polygon(Geometry *geom, face_valid = false; } } - curr_face.face_corners.append(corner); + geom->face_corners_.append(corner); + curr_face.corner_count_++; + + /* Skip whitespace to get to the next face corner. */ + line = drop_whitespace(line); } if (face_valid) { geom->face_elements_.append(curr_face); - geom->total_loops_ += curr_face.face_corners.size(); + geom->total_loops_ += curr_face.corner_count_; + } + else { + /* Remove just-added corners for the invalid face. */ + geom->face_corners_.resize(orig_corners_size); + geom->has_invalid_polys_ = true; } } static Geometry *geom_set_curve_type(Geometry *geom, const StringRef rest_line, const GlobalVertices &global_vertices, - const StringRef state_object_group, + const StringRef group_name, VertexIndexOffset &r_offsets, Vector> &r_all_geometries) { @@ -242,254 +211,409 @@ static Geometry *geom_set_curve_type(Geometry *geom, return geom; } geom = create_geometry( - geom, GEOM_CURVE, state_object_group, global_vertices, r_all_geometries, r_offsets); - geom->nurbs_element_.group_ = state_object_group; + geom, GEOM_CURVE, group_name, global_vertices, r_all_geometries, r_offsets); + geom->nurbs_element_.group_ = group_name; return geom; } -static void geom_set_curve_degree(Geometry *geom, const StringRef rest_line) +static void geom_set_curve_degree(Geometry *geom, const StringRef line) { - copy_string_to_int(rest_line, 3, geom->nurbs_element_.degree); + parse_int(line, 3, geom->nurbs_element_.degree); } static void geom_add_curve_vertex_indices(Geometry *geom, - const StringRef rest_line, + StringRef line, const GlobalVertices &global_vertices) { - Vector str_curv_split; - split_by_char(rest_line, ' ', str_curv_split); - /* Remove "0.0" and "1.0" from the strings. They are hardcoded. */ - str_curv_split.remove(0); - str_curv_split.remove(0); - geom->nurbs_element_.curv_indices.resize(str_curv_split.size()); - copy_string_to_int(str_curv_split, INT32_MAX, geom->nurbs_element_.curv_indices); - for (int &curv_index : geom->nurbs_element_.curv_indices) { + /* Curve lines always have "0.0" and "1.0", skip over them. */ + float dummy[2]; + line = parse_floats(line, 0, dummy, 2); + /* Parse indices. */ + while (!line.is_empty()) { + int index; + line = parse_int(line, INT32_MAX, index); + if (index == INT32_MAX) { + return; + } /* Always keep stored indices non-negative and zero-based. */ - curv_index += curv_index < 0 ? global_vertices.vertices.size() : -1; + index += index < 0 ? global_vertices.vertices.size() : -1; + geom->nurbs_element_.curv_indices.append(index); } } -static void geom_add_curve_parameters(Geometry *geom, const StringRef rest_line) +static void geom_add_curve_parameters(Geometry *geom, StringRef line) { - Vector str_parm_split; - split_by_char(rest_line, ' ', str_parm_split); - if (str_parm_split[0] != "u" && str_parm_split[0] != "v") { - std::cerr << "Surfaces are not supported:'" << str_parm_split[0] << "'" << std::endl; + line = drop_whitespace(line); + if (line.is_empty()) { + std::cerr << "Invalid OBJ curve parm line: '" << line << "'" << std::endl; + return; + } + if (line[0] != 'u') { + std::cerr << "OBJ curve surfaces are not supported: '" << line[0] << "'" << std::endl; return; } - str_parm_split.remove(0); - geom->nurbs_element_.parm.resize(str_parm_split.size()); - copy_string_to_float(str_parm_split, FLT_MAX, geom->nurbs_element_.parm); + line = line.drop_prefix(1); + + while (!line.is_empty()) { + float val; + line = parse_float(line, FLT_MAX, val); + if (val != FLT_MAX) { + geom->nurbs_element_.parm.append(val); + } + else { + std::cerr << "OBJ curve parm line has invalid number" << std::endl; + return; + } + } } -static void geom_update_object_group(const StringRef rest_line, std::string &r_state_object_group) +static void geom_update_group(const StringRef rest_line, std::string &r_group_name) { if (rest_line.find("off") != string::npos || rest_line.find("null") != string::npos || rest_line.find("default") != string::npos) { /* Set group for future elements like faces or curves to empty. */ - r_state_object_group = ""; + r_group_name = ""; return; } - r_state_object_group = rest_line; -} - -static void geom_update_polygon_material(Geometry *geom, - const StringRef rest_line, - std::string &r_state_material_name) -{ - /* Materials may repeat if faces are written without sorting. */ - geom->material_names_.add(string(rest_line)); - r_state_material_name = rest_line; + r_group_name = rest_line; } -static void geom_update_smooth_group(const StringRef rest_line, bool &r_state_shaded_smooth) +static void geom_update_smooth_group(StringRef line, bool &r_state_shaded_smooth) { + line = drop_whitespace(line); /* Some implementations use "0" and "null" too, in addition to "off". */ - if (rest_line != "0" && rest_line.find("off") == StringRef::not_found && - rest_line.find("null") == StringRef::not_found) { - int smooth = 0; - copy_string_to_int(rest_line, 0, smooth); - r_state_shaded_smooth = smooth != 0; - } - else { - /* The OBJ file explicitly set shading to off. */ + if (line == "0" || line.startswith("off") || line.startswith("null")) { r_state_shaded_smooth = false; + return; } + + int smooth = 0; + parse_int(line, 0, smooth); + r_state_shaded_smooth = smooth != 0; } -OBJParser::OBJParser(const OBJImportParams &import_params) : import_params_(import_params) +OBJParser::OBJParser(const OBJImportParams &import_params, size_t read_buffer_size = 64 * 1024) + : import_params_(import_params), read_buffer_size_(read_buffer_size) { - obj_file_.open(import_params_.filepath); - if (!obj_file_.good()) { + obj_file_ = BLI_fopen(import_params_.filepath, "rb"); + if (!obj_file_) { fprintf(stderr, "Cannot read from OBJ file:'%s'.\n", import_params_.filepath); return; } } +OBJParser::~OBJParser() +{ + if (obj_file_) { + fclose(obj_file_); + } +} + void OBJParser::parse(Vector> &r_all_geometries, GlobalVertices &r_global_vertices) { - if (!obj_file_.good()) { + if (!obj_file_) { return; } - string line; - /* Store vertex coordinates that belong to other Geometry instances. */ VertexIndexOffset offsets; - /* Non owning raw pointer to a Geometry. To be updated while creating a new Geometry. */ Geometry *curr_geom = create_geometry( nullptr, GEOM_MESH, "", r_global_vertices, r_all_geometries, offsets); - /* State-setting variables: if set, they remain the same for the remaining + /* State variables: once set, they remain the same for the remaining * elements in the object. */ bool state_shaded_smooth = false; - string state_object_group; + string state_group_name; + int state_group_index = -1; string state_material_name; + int state_material_index = -1; + + /* Read the input file in chunks. We need up to twice the possible chunk size, + * to possibly store remainder of the previous input line that got broken mid-chunk. */ + Array buffer(read_buffer_size_ * 2); + + size_t buffer_offset = 0; + size_t line_number = 0; + while (true) { + /* Read a chunk of input from the file. */ + size_t bytes_read = fread(buffer.data() + buffer_offset, 1, read_buffer_size_, obj_file_); + if (bytes_read == 0 && buffer_offset == 0) { + break; /* No more data to read. */ + } - while (std::getline(obj_file_, line)) { - /* Keep reading new lines if the last character is `\`. */ - /* Another way is to make a getline wrapper and use it in the while condition. */ - read_next_line(obj_file_, line); + /* Ensure buffer ends in a newline. */ + if (bytes_read < read_buffer_size_) { + if (bytes_read == 0 || buffer[buffer_offset + bytes_read - 1] != '\n') { + buffer[buffer_offset + bytes_read] = '\n'; + bytes_read++; + } + } - StringRef line_key, rest_line; - split_line_key_rest(line, line_key, rest_line); - if (line.empty() || rest_line.is_empty()) { - continue; + size_t buffer_end = buffer_offset + bytes_read; + if (buffer_end == 0) { + break; } - switch (line_key_str_to_enum(line_key)) { - case eOBJLineKey::V: { - geom_add_vertex(curr_geom, rest_line, r_global_vertices); - break; - } - case eOBJLineKey::VN: { - geom_add_vertex_normal(curr_geom, rest_line, r_global_vertices); - break; + + /* Find last newline. */ + size_t last_nl = buffer_end; + while (last_nl > 0) { + --last_nl; + if (buffer[last_nl] == '\n') { + if (last_nl < 1 || buffer[last_nl - 1] != '\\') + break; } - case eOBJLineKey::VT: { - geom_add_uv_vertex(rest_line, r_global_vertices); - break; + } + if (buffer[last_nl] != '\n') { + /* Whole line did not fit into our read buffer. Warn and exit. */ + fprintf(stderr, + "OBJ file contains a line #%zu that is too long (max. length %zu)\n", + line_number, + read_buffer_size_); + break; + } + ++last_nl; + + /* Parse the buffer (until last newline) that we have so far, + * line by line. */ + StringRef buffer_str{buffer.data(), (int64_t)last_nl}; + while (!buffer_str.is_empty()) { + StringRef line = read_next_line(buffer_str); + ++line_number; + if (line.is_empty()) + continue; + /* Most common things that start with 'v': vertices, normals, UVs. */ + if (line[0] == 'v') { + if (line.startswith("v ")) { + line = line.drop_prefix(2); + geom_add_vertex(curr_geom, line, r_global_vertices); + } + else if (line.startswith("vn ")) { + line = line.drop_prefix(3); + geom_add_vertex_normal(curr_geom, line, r_global_vertices); + } + else if (line.startswith("vt ")) { + line = line.drop_prefix(3); + geom_add_uv_vertex(line, r_global_vertices); + } } - case eOBJLineKey::F: { + /* Faces. */ + else if (line.startswith("f ")) { + line = line.drop_prefix(2); geom_add_polygon(curr_geom, - rest_line, + line, r_global_vertices, offsets, - state_material_name, - state_material_name, + state_material_index, + state_group_index, /* TODO was wrongly material name! */ state_shaded_smooth); - break; - } - case eOBJLineKey::L: { - geom_add_edge(curr_geom, rest_line, offsets, r_global_vertices); - break; - } - case eOBJLineKey::CSTYPE: { - curr_geom = geom_set_curve_type(curr_geom, - rest_line, - r_global_vertices, - state_object_group, - offsets, - r_all_geometries); - break; - } - case eOBJLineKey::DEG: { - geom_set_curve_degree(curr_geom, rest_line); - break; - } - case eOBJLineKey::CURV: { - geom_add_curve_vertex_indices(curr_geom, rest_line, r_global_vertices); - break; - } - case eOBJLineKey::PARM: { - geom_add_curve_parameters(curr_geom, rest_line); - break; - } - case eOBJLineKey::O: { + } + /* Faces. */ + else if (line.startswith("l ")) { + line = line.drop_prefix(2); + geom_add_edge(curr_geom, line, offsets, r_global_vertices); + } + /* Objects. */ + else if (line.startswith("o ")) { + line = line.drop_prefix(2); state_shaded_smooth = false; - state_object_group = ""; + state_group_name = ""; state_material_name = ""; curr_geom = create_geometry( - curr_geom, GEOM_MESH, rest_line, r_global_vertices, r_all_geometries, offsets); - break; - } - case eOBJLineKey::G: { - geom_update_object_group(rest_line, state_object_group); - break; - } - case eOBJLineKey::S: { - geom_update_smooth_group(rest_line, state_shaded_smooth); - break; - } - case eOBJLineKey::USEMTL: { - geom_update_polygon_material(curr_geom, rest_line, state_material_name); - break; - } - case eOBJLineKey::MTLLIB: { - mtl_libraries_.append(string(rest_line)); - break; - } - case eOBJLineKey::COMMENT: - break; - default: - std::cout << "Element not recognised: '" << line_key << "'" << std::endl; - break; + curr_geom, GEOM_MESH, line, r_global_vertices, r_all_geometries, offsets); + } + /* Groups. */ + else if (line.startswith("g ")) { + line = line.drop_prefix(2); + geom_update_group(line, state_group_name); + int new_index = curr_geom->group_indices_.size(); + state_group_index = curr_geom->group_indices_.lookup_or_add(state_group_name, new_index); + if (new_index == state_group_index) { + curr_geom->group_order_.append(state_group_name); + } + } + /* Smoothing groups. */ + else if (line.startswith("s ")) { + line = line.drop_prefix(2); + geom_update_smooth_group(line, state_shaded_smooth); + } + /* Materials and their libraries. */ + else if (line.startswith("usemtl ")) { + line = line.drop_prefix(7); + state_material_name = line; + int new_mat_index = curr_geom->material_indices_.size(); + state_material_index = curr_geom->material_indices_.lookup_or_add(state_material_name, + new_mat_index); + if (new_mat_index == state_material_index) { + curr_geom->material_order_.append(state_material_name); + } + } + else if (line.startswith("mtllib ")) { + line = line.drop_prefix(7); + mtl_libraries_.append(string(line)); + } + /* Comments. */ + else if (line.startswith("#")) { + /* Nothing to do. */ + } + /* Curve related things. */ + else if (line.startswith("cstype ")) { + line = line.drop_prefix(7); + curr_geom = geom_set_curve_type( + curr_geom, line, r_global_vertices, state_group_name, offsets, r_all_geometries); + } + else if (line.startswith("deg ")) { + line = line.drop_prefix(4); + geom_set_curve_degree(curr_geom, line); + } + else if (line.startswith("curv ")) { + line = line.drop_prefix(5); + geom_add_curve_vertex_indices(curr_geom, line, r_global_vertices); + } + else if (line.startswith("parm ")) { + line = line.drop_prefix(5); + geom_add_curve_parameters(curr_geom, line); + } + else if (line.startswith("end")) { + /* End of curve definition, nothing else to do. */ + } + else { + std::cout << "OBJ element not recognised: '" << line << "'" << std::endl; + } } + + /* We might have a line that was cut in the middle by the previous buffer; + * copy it over for next chunk reading. */ + size_t left_size = buffer_end - last_nl; + memmove(buffer.data(), buffer.data() + last_nl, left_size); + buffer_offset = left_size; } } -/** - * Skip all texture map options and get the filepath from a "map_" line. - */ -static StringRef skip_unsupported_options(StringRef line) +static eMTLSyntaxElement mtl_line_start_to_enum(StringRef &line) { - TextureMapOptions map_options; - StringRef last_option; - int64_t last_option_pos = 0; - - /* Find the last texture map option. */ - for (StringRef option : map_options.all_options()) { - const int64_t pos{line.find(option)}; - /* Equality (>=) takes care of finding an option in the beginning of the line. Avoid messing - * with signed-unsigned int comparison. */ - if (pos != StringRef::not_found && pos >= last_option_pos) { - last_option = option; - last_option_pos = pos; - } + if (line.startswith("map_Kd")) { + line = line.drop_prefix(6); + return eMTLSyntaxElement::map_Kd; } - - if (last_option.is_empty()) { - /* No option found, line is the filepath */ - return line; + if (line.startswith("map_Ks")) { + line = line.drop_prefix(6); + return eMTLSyntaxElement::map_Ks; + } + if (line.startswith("map_Ns")) { + line = line.drop_prefix(6); + return eMTLSyntaxElement::map_Ns; + } + if (line.startswith("map_d")) { + line = line.drop_prefix(5); + return eMTLSyntaxElement::map_d; } + if (line.startswith("refl")) { + line = line.drop_prefix(4); + return eMTLSyntaxElement::map_refl; + } + if (line.startswith("map_refl")) { + line = line.drop_prefix(8); + return eMTLSyntaxElement::map_refl; + } + if (line.startswith("map_Ke")) { + line = line.drop_prefix(6); + return eMTLSyntaxElement::map_Ke; + } + if (line.startswith("bump")) { + line = line.drop_prefix(4); + return eMTLSyntaxElement::map_Bump; + } + if (line.startswith("map_Bump") || line.startswith("map_bump")) { + line = line.drop_prefix(8); + return eMTLSyntaxElement::map_Bump; + } + return eMTLSyntaxElement::string; +} - /* Remove up to start of the last option + size of the last option + space after it. */ - line = line.drop_prefix(last_option_pos + last_option.size() + 1); - for (int i = 0; i < map_options.number_of_args(last_option); i++) { - const int64_t pos_space{line.find_first_of(' ')}; - if (pos_space != StringRef::not_found) { - BLI_assert(pos_space + 1 < line.size()); - line = line.drop_prefix(pos_space + 1); +static const std::pair unsupported_texture_options[] = { + {"-blendu ", 1}, + {"-blendv ", 1}, + {"-boost ", 1}, + {"-cc ", 1}, + {"-clamp ", 1}, + {"-imfchan ", 1}, + {"-mm ", 2}, + {"-t ", 3}, + {"-texres ", 1}, +}; + +static bool parse_texture_option(StringRef &line, MTLMaterial *material, tex_map_XX &tex_map) +{ + line = drop_whitespace(line); + if (line.startswith("-o ")) { + line = line.drop_prefix(3); + line = parse_floats(line, 0.0f, tex_map.translation, 3); + return true; + } + if (line.startswith("-s ")) { + line = line.drop_prefix(3); + line = parse_floats(line, 1.0f, tex_map.scale, 3); + return true; + } + if (line.startswith("-bm ")) { + line = line.drop_prefix(4); + line = parse_float(line, 0.0f, material->map_Bump_strength); + return true; + } + if (line.startswith("-type ")) { + line = line.drop_prefix(6); + line = drop_whitespace(line); + /* Only sphere is supported. */ + tex_map.projection_type = SHD_PROJ_SPHERE; + if (!line.startswith("sphere")) { + std::cerr << "OBJ import: only sphere MTL projection type is supported: '" << line << "'" + << std::endl; + } + line = drop_non_whitespace(line); + return true; + } + /* Check for unsupported options and skip them. */ + for (const auto &opt : unsupported_texture_options) { + if (line.startswith(opt.first)) { + /* Drop the option name. */ + line = line.drop_known_prefix(opt.first); + /* Drop the arguments. */ + for (int i = 0; i < opt.second; ++i) { + line = drop_whitespace(line); + line = drop_non_whitespace(line); + } + return true; } } - return line; + return false; } -/** - * Fix incoming texture map line keys for variations due to other exporters. - */ -static string fix_bad_map_keys(StringRef map_key) +static void parse_texture_map(StringRef line, MTLMaterial *material, const char *mtl_dir_path) { - string new_map_key(map_key); - if (map_key == "refl") { - new_map_key = "map_refl"; + bool is_map = line.startswith("map_"); + bool is_refl = line.startswith("refl"); + bool is_bump = line.startswith("bump"); + if (!is_map && !is_refl && !is_bump) { + return; + } + eMTLSyntaxElement key = mtl_line_start_to_enum(line); + if (key == eMTLSyntaxElement::string || !material->texture_maps.contains(key)) { + /* No supported texture map found. */ + std::cerr << "OBJ import: MTL texture map type not supported: '" << line << "'" << std::endl; + return; } - if (map_key.find("bump") != StringRef::not_found) { - /* Handles both "bump" and "map_Bump" */ - new_map_key = "map_Bump"; + tex_map_XX &tex_map = material->texture_maps.lookup(key); + tex_map.mtl_dir_path = mtl_dir_path; + + /* Parse texture map options. */ + while (parse_texture_option(line, material, tex_map)) { } - return new_map_key; + + /* What remains is the image path. */ + line = line.trim(); + tex_map.image_path = line; } Span OBJParser::mtl_libraries() const @@ -503,125 +627,72 @@ MTLParser::MTLParser(StringRef mtl_library, StringRefNull obj_filepath) BLI_split_dir_part(obj_filepath.data(), obj_file_dir, FILE_MAXDIR); BLI_path_join(mtl_file_path_, FILE_MAX, obj_file_dir, mtl_library.data(), NULL); BLI_split_dir_part(mtl_file_path_, mtl_dir_path_, FILE_MAXDIR); - mtl_file_.open(mtl_file_path_); - if (!mtl_file_.good()) { - fprintf(stderr, "Cannot read from MTL file:'%s'\n", mtl_file_path_); - return; - } } -void MTLParser::parse_and_store(Map> &r_mtl_materials) +void MTLParser::parse_and_store(Map> &r_materials) { - if (!mtl_file_.good()) { + size_t buffer_len; + void *buffer = BLI_file_read_text_as_mem(mtl_file_path_, 0, &buffer_len); + if (buffer == nullptr) { + fprintf(stderr, "OBJ import: cannot read from MTL file: '%s'\n", mtl_file_path_); return; } - string line; - MTLMaterial *current_mtlmaterial = nullptr; + MTLMaterial *material = nullptr; - while (std::getline(mtl_file_, line)) { - StringRef line_key, rest_line; - split_line_key_rest(line, line_key, rest_line); - if (line.empty() || rest_line.is_empty()) { + StringRef buffer_str{(const char *)buffer, (int64_t)buffer_len}; + while (!buffer_str.is_empty()) { + StringRef line = read_next_line(buffer_str); + if (line.is_empty()) continue; - } - /* Fix lower case/ incomplete texture map identifiers. */ - const string fixed_key = fix_bad_map_keys(line_key); - line_key = fixed_key; - - if (line_key == "newmtl") { - if (r_mtl_materials.remove_as(rest_line)) { - std::cerr << "Duplicate material found:'" << rest_line + if (line.startswith("newmtl ")) { + line = line.drop_prefix(7); + if (r_materials.remove_as(line)) { + std::cerr << "Duplicate material found:'" << line << "', using the last encountered Material definition." << std::endl; } - current_mtlmaterial = - r_mtl_materials.lookup_or_add(string(rest_line), std::make_unique()).get(); - } - else if (line_key == "Ns") { - copy_string_to_float(rest_line, 324.0f, current_mtlmaterial->Ns); - } - else if (line_key == "Ka") { - Vector str_ka_split; - split_by_char(rest_line, ' ', str_ka_split); - copy_string_to_float(str_ka_split, 0.0f, {current_mtlmaterial->Ka, 3}); - } - else if (line_key == "Kd") { - Vector str_kd_split; - split_by_char(rest_line, ' ', str_kd_split); - copy_string_to_float(str_kd_split, 0.8f, {current_mtlmaterial->Kd, 3}); - } - else if (line_key == "Ks") { - Vector str_ks_split; - split_by_char(rest_line, ' ', str_ks_split); - copy_string_to_float(str_ks_split, 0.5f, {current_mtlmaterial->Ks, 3}); - } - else if (line_key == "Ke") { - Vector str_ke_split; - split_by_char(rest_line, ' ', str_ke_split); - copy_string_to_float(str_ke_split, 0.0f, {current_mtlmaterial->Ke, 3}); - } - else if (line_key == "Ni") { - copy_string_to_float(rest_line, 1.45f, current_mtlmaterial->Ni); + material = r_materials.lookup_or_add(string(line), std::make_unique()).get(); } - else if (line_key == "d") { - copy_string_to_float(rest_line, 1.0f, current_mtlmaterial->d); - } - else if (line_key == "illum") { - copy_string_to_int(rest_line, 2, current_mtlmaterial->illum); - } - - /* Parse image textures. */ - else if (line_key.find("map_") != StringRef::not_found) { - /* TODO(@howardt): fix this. */ - eMTLSyntaxElement line_key_enum = mtl_line_key_str_to_enum(line_key); - if (line_key_enum == eMTLSyntaxElement::string || - !current_mtlmaterial->texture_maps.contains_as(line_key_enum)) { - /* No supported texture map found. */ - std::cerr << "Texture map type not supported:'" << line_key << "'" << std::endl; - continue; + else if (material != nullptr) { + if (line.startswith("Ns ")) { + line = line.drop_prefix(3); + parse_float(line, 324.0f, material->Ns); } - tex_map_XX &tex_map = current_mtlmaterial->texture_maps.lookup(line_key_enum); - Vector str_map_xx_split; - split_by_char(rest_line, ' ', str_map_xx_split); - - /* TODO(@ankitm): use `skip_unsupported_options` for parsing these options too? */ - const int64_t pos_o{str_map_xx_split.first_index_of_try("-o")}; - if (pos_o != -1 && pos_o + 3 < str_map_xx_split.size()) { - copy_string_to_float({str_map_xx_split[pos_o + 1], - str_map_xx_split[pos_o + 2], - str_map_xx_split[pos_o + 3]}, - 0.0f, - {tex_map.translation, 3}); - } - const int64_t pos_s{str_map_xx_split.first_index_of_try("-s")}; - if (pos_s != -1 && pos_s + 3 < str_map_xx_split.size()) { - copy_string_to_float({str_map_xx_split[pos_s + 1], - str_map_xx_split[pos_s + 2], - str_map_xx_split[pos_s + 3]}, - 1.0f, - {tex_map.scale, 3}); - } - /* Only specific to Normal Map node. */ - const int64_t pos_bm{str_map_xx_split.first_index_of_try("-bm")}; - if (pos_bm != -1 && pos_bm + 1 < str_map_xx_split.size()) { - copy_string_to_float( - str_map_xx_split[pos_bm + 1], 0.0f, current_mtlmaterial->map_Bump_strength); - } - const int64_t pos_projection{str_map_xx_split.first_index_of_try("-type")}; - if (pos_projection != -1 && pos_projection + 1 < str_map_xx_split.size()) { - /* Only Sphere is supported, so whatever the type is, set it to Sphere. */ - tex_map.projection_type = SHD_PROJ_SPHERE; - if (str_map_xx_split[pos_projection + 1] != "sphere") { - std::cerr << "Using projection type 'sphere', not:'" - << str_map_xx_split[pos_projection + 1] << "'." << std::endl; - } + else if (line.startswith("Ka ")) { + line = line.drop_prefix(3); + parse_floats(line, 0.0f, material->Ka, 3); + } + else if (line.startswith("Kd ")) { + line = line.drop_prefix(3); + parse_floats(line, 0.8f, material->Kd, 3); + } + else if (line.startswith("Ks ")) { + line = line.drop_prefix(3); + parse_floats(line, 0.5f, material->Ks, 3); + } + else if (line.startswith("Ke ")) { + line = line.drop_prefix(3); + parse_floats(line, 0.0f, material->Ke, 3); + } + else if (line.startswith("Ni ")) { + line = line.drop_prefix(3); + parse_float(line, 1.45f, material->Ni); + } + else if (line.startswith("d ")) { + line = line.drop_prefix(2); + parse_float(line, 1.0f, material->d); + } + else if (line.startswith("illum ")) { + line = line.drop_prefix(6); + parse_int(line, 2, material->illum); + } + else { + parse_texture_map(line, material, mtl_dir_path_); } - - /* Skip all unsupported options and arguments. */ - tex_map.image_path = string(skip_unsupported_options(rest_line)); - tex_map.mtl_dir_path = mtl_dir_path_; } } + + MEM_freeN(buffer); } } // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.hh b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.hh index 24d026d75e5..8093417fcda 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.hh +++ b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.hh @@ -18,14 +18,16 @@ namespace blender::io::obj { class OBJParser { private: const OBJImportParams &import_params_; - blender::fstream obj_file_; + FILE *obj_file_; Vector mtl_libraries_; + size_t read_buffer_size_; public: /** * Open OBJ file at the path given in import parameters. */ - OBJParser(const OBJImportParams &import_params); + OBJParser(const OBJImportParams &import_params, size_t read_buffer_size); + ~OBJParser(); /** * Read the OBJ file line by line and create OBJ Geometry instances. Also store all the vertex @@ -39,111 +41,6 @@ class OBJParser { Span mtl_libraries() const; }; -enum class eOBJLineKey { - V, - VN, - VT, - F, - L, - CSTYPE, - DEG, - CURV, - PARM, - O, - G, - S, - USEMTL, - MTLLIB, - COMMENT -}; - -constexpr eOBJLineKey line_key_str_to_enum(const std::string_view key_str) -{ - if (key_str == "v" || key_str == "V") { - return eOBJLineKey::V; - } - if (key_str == "vn" || key_str == "VN") { - return eOBJLineKey::VN; - } - if (key_str == "vt" || key_str == "VT") { - return eOBJLineKey::VT; - } - if (key_str == "f" || key_str == "F") { - return eOBJLineKey::F; - } - if (key_str == "l" || key_str == "L") { - return eOBJLineKey::L; - } - if (key_str == "cstype" || key_str == "CSTYPE") { - return eOBJLineKey::CSTYPE; - } - if (key_str == "deg" || key_str == "DEG") { - return eOBJLineKey::DEG; - } - if (key_str == "curv" || key_str == "CURV") { - return eOBJLineKey::CURV; - } - if (key_str == "parm" || key_str == "PARM") { - return eOBJLineKey::PARM; - } - if (key_str == "o" || key_str == "O") { - return eOBJLineKey::O; - } - if (key_str == "g" || key_str == "G") { - return eOBJLineKey::G; - } - if (key_str == "s" || key_str == "S") { - return eOBJLineKey::S; - } - if (key_str == "usemtl" || key_str == "USEMTL") { - return eOBJLineKey::USEMTL; - } - if (key_str == "mtllib" || key_str == "MTLLIB") { - return eOBJLineKey::MTLLIB; - } - if (key_str == "#") { - return eOBJLineKey::COMMENT; - } - return eOBJLineKey::COMMENT; -} - -/** - * All texture map options with number of arguments they accept. - */ -class TextureMapOptions { - private: - Map tex_map_options; - - public: - TextureMapOptions() - { - tex_map_options.add_new("-blendu", 1); - tex_map_options.add_new("-blendv", 1); - tex_map_options.add_new("-boost", 1); - tex_map_options.add_new("-mm", 2); - tex_map_options.add_new("-o", 3); - tex_map_options.add_new("-s", 3); - tex_map_options.add_new("-t", 3); - tex_map_options.add_new("-texres", 1); - tex_map_options.add_new("-clamp", 1); - tex_map_options.add_new("-bm", 1); - tex_map_options.add_new("-imfchan", 1); - } - - /** - * All valid option strings. - */ - Map::KeyIterator all_options() const - { - return tex_map_options.keys(); - } - - int number_of_args(StringRef option) const - { - return tex_map_options.lookup_as(std::string(option)); - } -}; - class MTLParser { private: char mtl_file_path_[FILE_MAX]; @@ -151,7 +48,6 @@ class MTLParser { * Directory in which the MTL file is found. */ char mtl_dir_path_[FILE_MAX]; - blender::fstream mtl_file_; public: /** @@ -162,6 +58,6 @@ class MTLParser { /** * Read MTL file(s) and add MTLMaterial instances to the given Map reference. */ - void parse_and_store(Map> &r_mtl_materials); + void parse_and_store(Map> &r_materials); }; } // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/importer/obj_import_mesh.cc b/source/blender/io/wavefront_obj/importer/obj_import_mesh.cc index 55b2873a3de..01a2d63927e 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_mesh.cc +++ b/source/blender/io/wavefront_obj/importer/obj_import_mesh.cc @@ -18,6 +18,7 @@ #include "BLI_math_vector.h" #include "BLI_set.hh" +#include "IO_wavefront_obj.h" #include "importer_mesh_utils.hh" #include "obj_import_mesh.hh" @@ -35,7 +36,7 @@ Object *MeshFromGeometry::create_mesh( } fixup_invalid_faces(); - const int64_t tot_verts_object{mesh_geometry_.vertex_indices_.size()}; + const int64_t tot_verts_object{mesh_geometry_.vertex_count_}; /* Total explicitly imported edges, not the ones belonging the polygons to be created. */ const int64_t tot_edges{mesh_geometry_.edges_.size()}; const int64_t tot_face_elems{mesh_geometry_.face_elements_.size()}; @@ -52,11 +53,13 @@ Object *MeshFromGeometry::create_mesh( create_normals(mesh); create_materials(bmain, materials, created_materials, obj); - bool verbose_validate = false; + if (import_params.validate_meshes || mesh_geometry_.has_invalid_polys_) { + bool verbose_validate = false; #ifdef DEBUG - verbose_validate = true; + verbose_validate = true; #endif - BKE_mesh_validate(mesh, verbose_validate, false); + BKE_mesh_validate(mesh, verbose_validate, false); + } transform_object(obj, import_params); /* FIXME: after 2.80; `mesh->flag` isn't copied by #BKE_mesh_nomain_to_mesh() */ @@ -73,9 +76,9 @@ void MeshFromGeometry::fixup_invalid_faces() for (int64_t face_idx = 0; face_idx < mesh_geometry_.face_elements_.size(); ++face_idx) { const PolyElem &curr_face = mesh_geometry_.face_elements_[face_idx]; - if (curr_face.face_corners.size() < 3) { + if (curr_face.corner_count_ < 3) { /* Skip and remove faces that have fewer than 3 corners. */ - mesh_geometry_.total_loops_ -= curr_face.face_corners.size(); + mesh_geometry_.total_loops_ -= curr_face.corner_count_; mesh_geometry_.face_elements_.remove_and_reorder(face_idx); continue; } @@ -84,12 +87,14 @@ void MeshFromGeometry::fixup_invalid_faces() * basically whether it has duplicate vertex indices. */ bool valid = true; Set used_verts; - for (const PolyCorner &corner : curr_face.face_corners) { - if (used_verts.contains(corner.vert_index)) { + for (int i = 0; i < curr_face.corner_count_; ++i) { + int corner_idx = curr_face.start_index_ + i; + int vertex_idx = mesh_geometry_.face_corners_[corner_idx].vert_index; + if (used_verts.contains(vertex_idx)) { valid = false; break; } - used_verts.add(corner.vert_index); + used_verts.add(vertex_idx); } if (valid) { continue; @@ -100,20 +105,22 @@ void MeshFromGeometry::fixup_invalid_faces() Vector face_verts; Vector face_uvs; Vector face_normals; - face_verts.reserve(curr_face.face_corners.size()); - face_uvs.reserve(curr_face.face_corners.size()); - face_normals.reserve(curr_face.face_corners.size()); - for (const PolyCorner &corner : curr_face.face_corners) { + face_verts.reserve(curr_face.corner_count_); + face_uvs.reserve(curr_face.corner_count_); + face_normals.reserve(curr_face.corner_count_); + for (int i = 0; i < curr_face.corner_count_; ++i) { + int corner_idx = curr_face.start_index_ + i; + const PolyCorner &corner = mesh_geometry_.face_corners_[corner_idx]; face_verts.append(corner.vert_index); face_normals.append(corner.vertex_normal_index); face_uvs.append(corner.uv_vert_index); } - std::string face_vertex_group = curr_face.vertex_group; - std::string face_material_name = curr_face.material_name; + int face_vertex_group = curr_face.vertex_group_index; + int face_material = curr_face.material_index; bool face_shaded_smooth = curr_face.shaded_smooth; /* Remove the invalid face. */ - mesh_geometry_.total_loops_ -= curr_face.face_corners.size(); + mesh_geometry_.total_loops_ -= curr_face.corner_count_; mesh_geometry_.face_elements_.remove_and_reorder(face_idx); Vector> new_faces = fixup_invalid_polygon(global_vertices_.vertices, face_verts); @@ -124,13 +131,14 @@ void MeshFromGeometry::fixup_invalid_faces() continue; } PolyElem new_face{}; - new_face.vertex_group = face_vertex_group; - new_face.material_name = face_material_name; + new_face.vertex_group_index = face_vertex_group; + new_face.material_index = face_material; new_face.shaded_smooth = face_shaded_smooth; - new_face.face_corners.reserve(face.size()); + new_face.start_index_ = mesh_geometry_.face_corners_.size(); + new_face.corner_count_ = face.size(); for (int idx : face) { BLI_assert(idx >= 0 && idx < face_verts.size()); - new_face.face_corners.append({face_verts[idx], face_uvs[idx], face_normals[idx]}); + mesh_geometry_.face_corners_.append({face_verts[idx], face_uvs[idx], face_normals[idx]}); } mesh_geometry_.face_elements_.append(new_face); mesh_geometry_.total_loops_ += face.size(); @@ -140,13 +148,14 @@ void MeshFromGeometry::fixup_invalid_faces() void MeshFromGeometry::create_vertices(Mesh *mesh) { - const int64_t tot_verts_object{mesh_geometry_.vertex_indices_.size()}; + const int tot_verts_object{mesh_geometry_.vertex_count_}; for (int i = 0; i < tot_verts_object; ++i) { - if (mesh_geometry_.vertex_indices_[i] < global_vertices_.vertices.size()) { - copy_v3_v3(mesh->mvert[i].co, global_vertices_.vertices[mesh_geometry_.vertex_indices_[i]]); + int vi = mesh_geometry_.vertex_start_ + i; + if (vi < global_vertices_.vertices.size()) { + copy_v3_v3(mesh->mvert[i].co, global_vertices_.vertices[vi]); } else { - std::cerr << "Vertex index:" << mesh_geometry_.vertex_indices_[i] + std::cerr << "Vertex index:" << vi << " larger than total vertices:" << global_vertices_.vertices.size() << " ." << std::endl; } @@ -158,7 +167,7 @@ void MeshFromGeometry::create_polys_loops(Object *obj, Mesh *mesh) /* Will not be used if vertex groups are not imported. */ mesh->dvert = nullptr; float weight = 0.0f; - const int64_t total_verts = mesh_geometry_.vertex_indices_.size(); + const int64_t total_verts = mesh_geometry_.vertex_count_; if (total_verts && mesh_geometry_.use_vertex_groups_) { mesh->dvert = static_cast( CustomData_add_layer(&mesh->vdata, CD_MDEFORMVERT, CD_CALLOC, nullptr, total_verts)); @@ -168,34 +177,32 @@ void MeshFromGeometry::create_polys_loops(Object *obj, Mesh *mesh) UNUSED_VARS(weight); } - /* Do not remove elements from the VectorSet since order of insertion is required. - * StringRef is fine since per-face deform group name outlives the VectorSet. */ - VectorSet group_names; const int64_t tot_face_elems{mesh->totpoly}; int tot_loop_idx = 0; for (int poly_idx = 0; poly_idx < tot_face_elems; ++poly_idx) { const PolyElem &curr_face = mesh_geometry_.face_elements_[poly_idx]; - if (curr_face.face_corners.size() < 3) { + if (curr_face.corner_count_ < 3) { /* Don't add single vertex face, or edges. */ std::cerr << "Face with less than 3 vertices found, skipping." << std::endl; continue; } MPoly &mpoly = mesh->mpoly[poly_idx]; - mpoly.totloop = curr_face.face_corners.size(); + mpoly.totloop = curr_face.corner_count_; mpoly.loopstart = tot_loop_idx; if (curr_face.shaded_smooth) { mpoly.flag |= ME_SMOOTH; } - mpoly.mat_nr = mesh_geometry_.material_names_.index_of_try(curr_face.material_name); + mpoly.mat_nr = curr_face.material_index; /* Importing obj files without any materials would result in negative indices, which is not * supported. */ if (mpoly.mat_nr < 0) { mpoly.mat_nr = 0; } - for (const PolyCorner &curr_corner : curr_face.face_corners) { + for (int idx = 0; idx < curr_face.corner_count_; ++idx) { + const PolyCorner &curr_corner = mesh_geometry_.face_corners_[curr_face.start_index_ + idx]; MLoop &mloop = mesh->mloop[tot_loop_idx]; tot_loop_idx++; mloop.v = curr_corner.vert_index; @@ -212,23 +219,17 @@ void MeshFromGeometry::create_polys_loops(Object *obj, Mesh *mesh) MEM_callocN(sizeof(MDeformWeight), "OBJ Import Deform Weight")); } /* Every vertex in a face is assigned the same deform group. */ - int64_t pos_name{group_names.index_of_try(curr_face.vertex_group)}; - if (pos_name == -1) { - group_names.add_new(curr_face.vertex_group); - pos_name = group_names.size() - 1; - } - BLI_assert(pos_name >= 0); + int group_idx = curr_face.vertex_group_index; /* Deform group number (def_nr) must behave like an index into the names' list. */ - *(def_vert.dw) = {static_cast(pos_name), weight}; + *(def_vert.dw) = {static_cast(group_idx), weight}; } } if (!mesh->dvert) { return; } - /* Add deform group(s) to the object's defbase. */ - for (StringRef name : group_names) { - /* Adding groups in this order assumes that def_nr is an index into the names' list. */ + /* Add deform group names. */ + for (const std::string &name : mesh_geometry_.group_order_) { BKE_object_defgroup_add_name(obj, name.data()); } } @@ -236,7 +237,7 @@ void MeshFromGeometry::create_polys_loops(Object *obj, Mesh *mesh) void MeshFromGeometry::create_edges(Mesh *mesh) { const int64_t tot_edges{mesh_geometry_.edges_.size()}; - const int64_t total_verts{mesh_geometry_.vertex_indices_.size()}; + const int64_t total_verts{mesh_geometry_.vertex_count_}; UNUSED_VARS_NDEBUG(total_verts); for (int i = 0; i < tot_edges; ++i) { const MEdge &src_edge = mesh_geometry_.edges_[i]; @@ -263,7 +264,8 @@ void MeshFromGeometry::create_uv_verts(Mesh *mesh) int tot_loop_idx = 0; for (const PolyElem &curr_face : mesh_geometry_.face_elements_) { - for (const PolyCorner &curr_corner : curr_face.face_corners) { + for (int idx = 0; idx < curr_face.corner_count_; ++idx) { + const PolyCorner &curr_corner = mesh_geometry_.face_corners_[curr_face.start_index_ + idx]; if (curr_corner.uv_vert_index >= 0 && curr_corner.uv_vert_index < global_vertices_.uv_vertices.size()) { const float2 &mluv_src = global_vertices_.uv_vertices[curr_corner.uv_vert_index]; @@ -317,7 +319,7 @@ void MeshFromGeometry::create_materials( Map &created_materials, Object *obj) { - for (const std::string &name : mesh_geometry_.material_names_) { + for (const std::string &name : mesh_geometry_.material_order_) { Material *mat = get_or_create_material(bmain, name, materials, created_materials); if (mat == nullptr) { continue; @@ -340,7 +342,8 @@ void MeshFromGeometry::create_normals(Mesh *mesh) MEM_malloc_arrayN(mesh_geometry_.total_loops_, sizeof(float[3]), __func__)); int tot_loop_idx = 0; for (const PolyElem &curr_face : mesh_geometry_.face_elements_) { - for (const PolyCorner &curr_corner : curr_face.face_corners) { + for (int idx = 0; idx < curr_face.corner_count_; ++idx) { + const PolyCorner &curr_corner = mesh_geometry_.face_corners_[curr_face.start_index_ + idx]; int n_index = curr_corner.vertex_normal_index; float3 normal(0, 0, 0); if (n_index >= 0) { diff --git a/source/blender/io/wavefront_obj/importer/obj_import_mesh.hh b/source/blender/io/wavefront_obj/importer/obj_import_mesh.hh index 2a838215421..7cc7ed25ad1 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_mesh.hh +++ b/source/blender/io/wavefront_obj/importer/obj_import_mesh.hh @@ -45,11 +45,8 @@ class MeshFromGeometry : NonMovable, NonCopyable { void fixup_invalid_faces(); void create_vertices(Mesh *mesh); /** - * Create polygons for the Mesh, set smooth shading flag, deform group name, - * assigned material also. - * - * It must receive all polygons to be added to the mesh. - * Remove holes from polygons before * calling this. + * Create polygons for the Mesh, set smooth shading flags, deform group names, + * Materials. */ void create_polys_loops(Object *obj, Mesh *mesh); /** diff --git a/source/blender/io/wavefront_obj/importer/obj_import_mtl.cc b/source/blender/io/wavefront_obj/importer/obj_import_mtl.cc index 88a8c07e325..f2a8941e8a7 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_mtl.cc +++ b/source/blender/io/wavefront_obj/importer/obj_import_mtl.cc @@ -13,12 +13,13 @@ #include "DNA_material_types.h" #include "DNA_node_types.h" +#include "IO_string_utils.hh" + #include "NOD_shader.h" /* TODO: move eMTLSyntaxElement out of following file into a more neutral place */ #include "obj_export_io.hh" #include "obj_import_mtl.hh" -#include "parser_string_utils.hh" namespace blender::io::obj { @@ -96,13 +97,16 @@ static bool load_texture_image(Main *bmain, const tex_map_XX &tex_map, bNode *r_ return true; } /* Try removing quotes. */ - std::string no_quote_path{replace_all_occurences(tex_path, "\"", "")}; + std::string no_quote_path{tex_path}; + auto end_pos = std::remove(no_quote_path.begin(), no_quote_path.end(), '"'); + no_quote_path.erase(end_pos, no_quote_path.end()); if (no_quote_path != tex_path && load_texture_image_at_path(bmain, tex_map, r_node, no_quote_path)) { return true; } /* Try replacing underscores with spaces. */ - std::string no_underscore_path{replace_all_occurences(no_quote_path, "_", " ")}; + std::string no_underscore_path{no_quote_path}; + std::replace(no_underscore_path.begin(), no_underscore_path.end(), '_', ' '); if (no_underscore_path != no_quote_path && no_underscore_path != tex_path && load_texture_image_at_path(bmain, tex_map, r_node, no_underscore_path)) { return true; diff --git a/source/blender/io/wavefront_obj/importer/obj_import_mtl.hh b/source/blender/io/wavefront_obj/importer/obj_import_mtl.hh index 4b7827b2035..74bc9f21bc4 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_mtl.hh +++ b/source/blender/io/wavefront_obj/importer/obj_import_mtl.hh @@ -90,29 +90,4 @@ class ShaderNodetreeWrap { void add_image_textures(Main *bmain, Material *mat); }; -constexpr eMTLSyntaxElement mtl_line_key_str_to_enum(const std::string_view key_str) -{ - if (key_str == "map_Kd") { - return eMTLSyntaxElement::map_Kd; - } - if (key_str == "map_Ks") { - return eMTLSyntaxElement::map_Ks; - } - if (key_str == "map_Ns") { - return eMTLSyntaxElement::map_Ns; - } - if (key_str == "map_d") { - return eMTLSyntaxElement::map_d; - } - if (key_str == "refl" || key_str == "map_refl") { - return eMTLSyntaxElement::map_refl; - } - if (key_str == "map_Ke") { - return eMTLSyntaxElement::map_Ke; - } - if (key_str == "map_Bump" || key_str == "bump") { - return eMTLSyntaxElement::map_Bump; - } - return eMTLSyntaxElement::string; -} } // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/importer/obj_import_objects.hh b/source/blender/io/wavefront_obj/importer/obj_import_objects.hh index c6ce7d3c434..b67ba46af03 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_objects.hh +++ b/source/blender/io/wavefront_obj/importer/obj_import_objects.hh @@ -8,6 +8,7 @@ #include "BKE_lib_id.h" +#include "BLI_map.hh" #include "BLI_math_vec_types.hh" #include "BLI_vector.hh" #include "BLI_vector_set.hh" @@ -61,10 +62,11 @@ struct PolyCorner { }; struct PolyElem { - std::string vertex_group; - std::string material_name; + int vertex_group_index = -1; + int material_index = -1; bool shaded_smooth = false; - Vector face_corners; + int start_index_ = 0; + int corner_count_ = 0; }; /** @@ -93,15 +95,20 @@ enum eGeometryType { struct Geometry { eGeometryType geom_type_ = GEOM_MESH; std::string geometry_name_; - VectorSet material_names_; - /** - * Indices in the vector range from zero to total vertices in a geometry. - * Values range from zero to total coordinates in the global list. - */ - Vector vertex_indices_; + Map group_indices_; + Vector group_order_; + Map material_indices_; + Vector material_order_; + + int vertex_start_ = 0; + int vertex_count_ = 0; /** Edges written in the file in addition to (or even without polygon) elements. */ Vector edges_; + + Vector face_corners_; Vector face_elements_; + + bool has_invalid_polys_ = false; bool has_vertex_normals_ = false; bool use_vertex_groups_ = false; NurbsElement nurbs_element_; diff --git a/source/blender/io/wavefront_obj/importer/obj_importer.cc b/source/blender/io/wavefront_obj/importer/obj_importer.cc index 631ddcc5cf4..c21d2d9583c 100644 --- a/source/blender/io/wavefront_obj/importer/obj_importer.cc +++ b/source/blender/io/wavefront_obj/importer/obj_importer.cc @@ -42,6 +42,12 @@ static void geometry_to_blender_objects( BKE_view_layer_base_deselect_all(view_layer); LayerCollection *lc = BKE_layer_collection_get_active(view_layer); + /* Don't do collection syncs for each object, will do once after the loop. */ + BKE_layer_collection_resync_forbid(); + + /* Create all the objects. */ + Vector objects; + objects.reserve(all_geometries.size()); for (const std::unique_ptr &geometry : all_geometries) { Object *obj = nullptr; if (geometry->geom_type_ == GEOM_MESH) { @@ -54,17 +60,25 @@ static void geometry_to_blender_objects( } if (obj != nullptr) { BKE_collection_object_add(bmain, lc->collection, obj); - Base *base = BKE_view_layer_base_find(view_layer, obj); - /* TODO: is setting active needed? */ - BKE_view_layer_base_select_and_set_active(view_layer, base); - - DEG_id_tag_update(&lc->collection->id, ID_RECALC_COPY_ON_WRITE); - DEG_id_tag_update_ex(bmain, - &obj->id, - ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY | ID_RECALC_ANIMATION | - ID_RECALC_BASE_FLAGS); + objects.append(obj); } } + + /* Sync the collection after all objects are created. */ + BKE_layer_collection_resync_allow(); + BKE_main_collection_sync(bmain); + + /* After collection sync, select objects in the view layer and do DEG updates. */ + for (Object *obj : objects) { + Base *base = BKE_view_layer_base_find(view_layer, obj); + BKE_view_layer_base_select_and_set_active(view_layer, base); + + DEG_id_tag_update(&lc->collection->id, ID_RECALC_COPY_ON_WRITE); + int flags = ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY | ID_RECALC_ANIMATION | + ID_RECALC_BASE_FLAGS; + DEG_id_tag_update_ex(bmain, &obj->id, flags); + } + DEG_id_tag_update(&scene->id, ID_RECALC_BASE_FLAGS); DEG_relations_tag_update(bmain); } @@ -81,7 +95,8 @@ void importer_main(bContext *C, const OBJImportParams &import_params) void importer_main(Main *bmain, Scene *scene, ViewLayer *view_layer, - const OBJImportParams &import_params) + const OBJImportParams &import_params, + size_t read_buffer_size) { /* List of Geometry instances to be parsed from OBJ file. */ Vector> all_geometries; @@ -91,7 +106,7 @@ void importer_main(Main *bmain, Map> materials; Map created_materials; - OBJParser obj_parser{import_params}; + OBJParser obj_parser{import_params, read_buffer_size}; obj_parser.parse(all_geometries, global_vertices); for (StringRef mtl_library : obj_parser.mtl_libraries()) { diff --git a/source/blender/io/wavefront_obj/importer/obj_importer.hh b/source/blender/io/wavefront_obj/importer/obj_importer.hh index fd83117ebc6..35f401d7cb0 100644 --- a/source/blender/io/wavefront_obj/importer/obj_importer.hh +++ b/source/blender/io/wavefront_obj/importer/obj_importer.hh @@ -17,6 +17,7 @@ void importer_main(bContext *C, const OBJImportParams &import_params); void importer_main(Main *bmain, Scene *scene, ViewLayer *view_layer, - const OBJImportParams &import_params); + const OBJImportParams &import_params, + size_t read_buffer_size = 64 * 1024); } // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/importer/parser_string_utils.cc b/source/blender/io/wavefront_obj/importer/parser_string_utils.cc deleted file mode 100644 index 6671a86f5ee..00000000000 --- a/source/blender/io/wavefront_obj/importer/parser_string_utils.cc +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ - -#include -#include -#include - -#include "BLI_math_vec_types.hh" -#include "BLI_span.hh" -#include "BLI_string_ref.hh" -#include "BLI_vector.hh" - -#include "parser_string_utils.hh" - -/* Note: these OBJ parser helper functions are planned to get fairly large - * changes "soon", so don't read too much into current implementation... */ - -namespace blender::io::obj { -using std::string; - -void read_next_line(std::fstream &file, string &r_line) -{ - std::string new_line; - while (file.good() && !r_line.empty() && r_line.back() == '\\') { - new_line.clear(); - const bool ok = static_cast(std::getline(file, new_line)); - /* Remove the last backslash character. */ - r_line.pop_back(); - r_line.append(new_line); - if (!ok || new_line.empty()) { - return; - } - } -} - -void split_line_key_rest(const StringRef line, StringRef &r_line_key, StringRef &r_rest_line) -{ - if (line.is_empty()) { - return; - } - - const int64_t pos_split{line.find_first_of(' ')}; - if (pos_split == StringRef::not_found) { - /* Use the first character if no space is found in the line. It's usually a comment like: - * #This is a comment. */ - r_line_key = line.substr(0, 1); - } - else { - r_line_key = line.substr(0, pos_split); - } - - /* Eat the delimiter also using "+ 1". */ - r_rest_line = line.drop_prefix(r_line_key.size() + 1); - if (r_rest_line.is_empty()) { - return; - } - - /* Remove any leading spaces, trailing spaces & \r character, if any. */ - const int64_t leading_space{r_rest_line.find_first_not_of(' ')}; - if (leading_space != StringRef::not_found) { - r_rest_line = r_rest_line.drop_prefix(leading_space); - } - - /* Another way is to do a test run before the actual parsing to find the newline - * character and use it in the getline. */ - const int64_t carriage_return{r_rest_line.find_first_of('\r')}; - if (carriage_return != StringRef::not_found) { - r_rest_line = r_rest_line.substr(0, carriage_return + 1); - } - - const int64_t trailing_space{r_rest_line.find_last_not_of(' ')}; - if (trailing_space != StringRef::not_found) { - /* The position is of a character that is not ' ', so count of characters is position + 1. */ - r_rest_line = r_rest_line.substr(0, trailing_space + 1); - } -} - -void split_by_char(StringRef in_string, const char delimiter, Vector &r_out_list) -{ - r_out_list.clear(); - - while (!in_string.is_empty()) { - const int64_t pos_delim{in_string.find_first_of(delimiter)}; - const int64_t word_len = pos_delim == StringRef::not_found ? in_string.size() : pos_delim; - - StringRef word{in_string.data(), word_len}; - if (!word.is_empty() && !(word == " " && !(word[0] == '\0'))) { - r_out_list.append(word); - } - if (pos_delim == StringRef::not_found) { - return; - } - /* Skip the word already stored. */ - in_string = in_string.drop_prefix(word_len); - /* Skip all delimiters. */ - const int64_t pos_non_delim = in_string.find_first_not_of(delimiter); - if (pos_non_delim == StringRef::not_found) { - return; - } - in_string = in_string.drop_prefix(std::min(pos_non_delim, in_string.size())); - } -} - -void copy_string_to_float(StringRef src, const float fallback_value, float &r_dst) -{ - try { - r_dst = std::stof(string(src)); - } - catch (const std::invalid_argument &inv_arg) { - std::cerr << "Bad conversion to float:'" << inv_arg.what() << "':'" << src << "'" << std::endl; - r_dst = fallback_value; - } - catch (const std::out_of_range &out_of_range) { - std::cerr << "Out of range for float:'" << out_of_range.what() << ":'" << src << "'" - << std::endl; - r_dst = fallback_value; - } -} - -void copy_string_to_float(Span src, - const float fallback_value, - MutableSpan r_dst) -{ - for (int i = 0; i < r_dst.size(); ++i) { - if (i < src.size()) { - copy_string_to_float(src[i], fallback_value, r_dst[i]); - } - else { - r_dst[i] = fallback_value; - } - } -} - -void copy_string_to_int(StringRef src, const int fallback_value, int &r_dst) -{ - try { - r_dst = std::stoi(string(src)); - } - catch (const std::invalid_argument &inv_arg) { - std::cerr << "Bad conversion to int:'" << inv_arg.what() << "':'" << src << "'" << std::endl; - r_dst = fallback_value; - } - catch (const std::out_of_range &out_of_range) { - std::cerr << "Out of range for int:'" << out_of_range.what() << ":'" << src << "'" - << std::endl; - r_dst = fallback_value; - } -} - -void copy_string_to_int(Span src, const int fallback_value, MutableSpan r_dst) -{ - for (int i = 0; i < r_dst.size(); ++i) { - if (i < src.size()) { - copy_string_to_int(src[i], fallback_value, r_dst[i]); - } - else { - r_dst[i] = fallback_value; - } - } -} - -std::string replace_all_occurences(StringRef original, StringRef to_remove, StringRef to_add) -{ - std::string clean{original}; - while (true) { - const std::string::size_type pos = clean.find(to_remove); - if (pos == std::string::npos) { - break; - } - clean.replace(pos, to_add.size(), to_add); - } - return clean; -} - -} // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/importer/parser_string_utils.hh b/source/blender/io/wavefront_obj/importer/parser_string_utils.hh deleted file mode 100644 index 62cfbebccf3..00000000000 --- a/source/blender/io/wavefront_obj/importer/parser_string_utils.hh +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ - -namespace blender::io::obj { - -/* Note: these OBJ parser helper functions are planned to get fairly large - * changes "soon", so don't read too much into current implementation... */ - -/** - * Store multiple lines separated by an escaped newline character: `\\n`. - * Use this before doing any parse operations on the read string. - */ -void read_next_line(std::fstream &file, std::string &r_line); -/** - * Split a line string into the first word (key) and the rest of the line. - * Also remove leading & trailing spaces as well as `\r` carriage return - * character if present. - */ -void split_line_key_rest(StringRef line, StringRef &r_line_key, StringRef &r_rest_line); -/** - * Split the given string by the delimiter and fill the given vector. - * If an intermediate string is empty, or space or null character, it is not appended to the - * vector. - */ -void split_by_char(StringRef in_string, const char delimiter, Vector &r_out_list); -/** - * Convert the given string to float and assign it to the destination value. - * - * If the string cannot be converted to a float, the fallback value is used. - */ -void copy_string_to_float(StringRef src, const float fallback_value, float &r_dst); -/** - * Convert all members of the Span of strings to floats and assign them to the float - * array members. Usually used for values like coordinates. - * - * If a string cannot be converted to a float, the fallback value is used. - */ -void copy_string_to_float(Span src, - const float fallback_value, - MutableSpan r_dst); -/** - * Convert the given string to int and assign it to the destination value. - * - * If the string cannot be converted to an integer, the fallback value is used. - */ -void copy_string_to_int(StringRef src, const int fallback_value, int &r_dst); -/** - * Convert the given strings to ints and fill the destination int buffer. - * - * If a string cannot be converted to an integer, the fallback value is used. - */ -void copy_string_to_int(Span src, const int fallback_value, MutableSpan r_dst); -std::string replace_all_occurences(StringRef original, StringRef to_remove, StringRef to_add); - -} // namespace blender::io::obj diff --git a/source/blender/io/wavefront_obj/tests/obj_importer_tests.cc b/source/blender/io/wavefront_obj/tests/obj_importer_tests.cc index 611e0cbb209..3d34fb6f9c6 100644 --- a/source/blender/io/wavefront_obj/tests/obj_importer_tests.cc +++ b/source/blender/io/wavefront_obj/tests/obj_importer_tests.cc @@ -60,7 +60,8 @@ class obj_importer_test : public BlendfileLoadingBaseTest { std::string obj_path = blender::tests::flags_test_asset_dir() + "/io_tests/obj/" + path; strncpy(params.filepath, obj_path.c_str(), FILE_MAX - 1); - importer_main(bfile->main, bfile->curscene, bfile->cur_view_layer, params); + const size_t read_buffer_size = 650; + importer_main(bfile->main, bfile->curscene, bfile->cur_view_layer, params, read_buffer_size); depsgraph_create(DAG_EVAL_VIEWPORT); diff --git a/source/blender/io/wavefront_obj/tests/obj_mtl_parser_tests.cc b/source/blender/io/wavefront_obj/tests/obj_mtl_parser_tests.cc new file mode 100644 index 00000000000..176d32a0be4 --- /dev/null +++ b/source/blender/io/wavefront_obj/tests/obj_mtl_parser_tests.cc @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include + +#include "testing/testing.h" + +#include "obj_import_file_reader.hh" + +namespace blender::io::obj { + +class obj_mtl_parser_test : public testing::Test { + public: + void check(const char *file, const MTLMaterial *expect, size_t expect_count) + { + std::string obj_dir = blender::tests::flags_test_asset_dir() + "/io_tests/obj/"; + MTLParser parser(file, obj_dir + "dummy.obj"); + Map> materials; + parser.parse_and_store(materials); + + for (int i = 0; i < expect_count; ++i) { + const MTLMaterial &exp = expect[i]; + if (!materials.contains(exp.name)) { + fprintf(stderr, "Material '%s' was expected in parsed result\n", exp.name.c_str()); + ADD_FAILURE(); + continue; + } + const MTLMaterial &got = *materials.lookup(exp.name); + const float tol = 0.0001f; + EXPECT_V3_NEAR(exp.Ka, got.Ka, tol); + EXPECT_V3_NEAR(exp.Kd, got.Kd, tol); + EXPECT_V3_NEAR(exp.Ks, got.Ks, tol); + EXPECT_V3_NEAR(exp.Ke, got.Ke, tol); + EXPECT_NEAR(exp.Ns, got.Ns, tol); + EXPECT_NEAR(exp.Ni, got.Ni, tol); + EXPECT_NEAR(exp.d, got.d, tol); + EXPECT_NEAR(exp.map_Bump_strength, got.map_Bump_strength, tol); + EXPECT_EQ(exp.illum, got.illum); + for (const auto &it : exp.texture_maps.items()) { + const tex_map_XX &exp_tex = it.value; + const tex_map_XX &got_tex = got.texture_maps.lookup(it.key); + EXPECT_STREQ(exp_tex.image_path.c_str(), got_tex.image_path.c_str()); + EXPECT_V3_NEAR(exp_tex.translation, got_tex.translation, tol); + EXPECT_V3_NEAR(exp_tex.scale, got_tex.scale, tol); + EXPECT_EQ(exp_tex.projection_type, got_tex.projection_type); + } + } + EXPECT_EQ(materials.size(), expect_count); + } +}; + +TEST_F(obj_mtl_parser_test, cube) +{ + MTLMaterial mat; + mat.name = "red"; + mat.Ka = {0.2f, 0.2f, 0.2f}; + mat.Kd = {1, 0, 0}; + check("cube.mtl", &mat, 1); +} + +TEST_F(obj_mtl_parser_test, all_objects) +{ + MTLMaterial mat[7]; + for (auto &m : mat) { + m.Ka = {1, 1, 1}; + m.Ks = {0.5f, 0.5f, 0.5f}; + m.Ke = {0, 0, 0}; + m.Ns = 250; + m.Ni = 1; + m.d = 1; + m.illum = 2; + } + mat[0].name = "Blue"; + mat[0].Kd = {0, 0, 1}; + mat[1].name = "BlueDark"; + mat[1].Kd = {0, 0, 0.5f}; + mat[2].name = "Green"; + mat[2].Kd = {0, 1, 0}; + mat[3].name = "GreenDark"; + mat[3].Kd = {0, 0.5f, 0}; + mat[4].name = "Material"; + mat[4].Kd = {0.8f, 0.8f, 0.8f}; + mat[5].name = "Red"; + mat[5].Kd = {1, 0, 0}; + mat[6].name = "RedDark"; + mat[6].Kd = {0.5f, 0, 0}; + check("all_objects.mtl", mat, ARRAY_SIZE(mat)); +} + +TEST_F(obj_mtl_parser_test, materials) +{ + MTLMaterial mat[5]; + mat[0].name = "no_textures_red"; + mat[0].Ka = {0.3f, 0.3f, 0.3f}; + mat[0].Kd = {0.8f, 0.3f, 0.1f}; + mat[0].Ns = 5.624998f; + + mat[1].name = "four_maps"; + mat[1].Ka = {1, 1, 1}; + mat[1].Kd = {0.8f, 0.8f, 0.8f}; + mat[1].Ks = {0.5f, 0.5f, 0.5f}; + mat[1].Ke = {0, 0, 0}; + mat[1].Ns = 1000; + mat[1].Ni = 1.45f; + mat[1].d = 1; + mat[1].illum = 2; + mat[1].map_Bump_strength = 1; + { + tex_map_XX &kd = mat[1].tex_map_of_type(eMTLSyntaxElement::map_Kd); + kd.image_path = "texture.png"; + tex_map_XX &ns = mat[1].tex_map_of_type(eMTLSyntaxElement::map_Ns); + ns.image_path = "sometexture_Roughness.png"; + tex_map_XX &refl = mat[1].tex_map_of_type(eMTLSyntaxElement::map_refl); + refl.image_path = "sometexture_Metallic.png"; + tex_map_XX &bump = mat[1].tex_map_of_type(eMTLSyntaxElement::map_Bump); + bump.image_path = "sometexture_Normal.png"; + } + + mat[2].name = "Clay"; + mat[2].Ka = {1, 1, 1}; + mat[2].Kd = {0.8f, 0.682657f, 0.536371f}; + mat[2].Ks = {0.5f, 0.5f, 0.5f}; + mat[2].Ke = {0, 0, 0}; + mat[2].Ns = 440.924042f; + mat[2].Ni = 1.45f; + mat[2].d = 1; + mat[2].illum = 2; + + mat[3].name = "Hat"; + mat[3].Ka = {1, 1, 1}; + mat[3].Kd = {0.8f, 0.8f, 0.8f}; + mat[3].Ks = {0.5f, 0.5f, 0.5f}; + mat[3].Ns = 800; + mat[3].map_Bump_strength = 0.5f; + { + tex_map_XX &kd = mat[3].tex_map_of_type(eMTLSyntaxElement::map_Kd); + kd.image_path = "someHatTexture_BaseColor.jpg"; + tex_map_XX &ns = mat[3].tex_map_of_type(eMTLSyntaxElement::map_Ns); + ns.image_path = "someHatTexture_Roughness.jpg"; + tex_map_XX &refl = mat[3].tex_map_of_type(eMTLSyntaxElement::map_refl); + refl.image_path = "someHatTexture_Metalness.jpg"; + tex_map_XX &bump = mat[3].tex_map_of_type(eMTLSyntaxElement::map_Bump); + bump.image_path = "someHatTexture_Normal.jpg"; + } + + mat[4].name = "Parser_Test"; + mat[4].Ka = {0.1f, 0.2f, 0.3f}; + mat[4].Kd = {0.4f, 0.5f, 0.6f}; + mat[4].Ks = {0.7f, 0.8f, 0.9f}; + mat[4].illum = 6; + mat[4].Ns = 15.5; + mat[4].Ni = 1.5; + mat[4].d = 0.5; + mat[4].map_Bump_strength = 0.1f; + { + tex_map_XX &kd = mat[4].tex_map_of_type(eMTLSyntaxElement::map_Kd); + kd.image_path = "sometex_d.png"; + tex_map_XX &ns = mat[4].tex_map_of_type(eMTLSyntaxElement::map_Ns); + ns.image_path = "sometex_ns.psd"; + tex_map_XX &refl = mat[4].tex_map_of_type(eMTLSyntaxElement::map_refl); + refl.image_path = "clouds.tiff"; + refl.scale = {1.5f, 2.5f, 3.5f}; + refl.translation = {4.5f, 5.5f, 6.5f}; + refl.projection_type = SHD_PROJ_SPHERE; + tex_map_XX &bump = mat[4].tex_map_of_type(eMTLSyntaxElement::map_Bump); + bump.image_path = "somebump.tga"; + bump.scale = {3, 4, 5}; + } + + check("materials.mtl", mat, ARRAY_SIZE(mat)); +} + +} // namespace blender::io::obj -- cgit v1.2.3