diff options
author | Jacques Lucke <jacques@blender.org> | 2021-08-22 13:21:52 +0300 |
---|---|---|
committer | Jacques Lucke <jacques@blender.org> | 2021-08-22 13:21:52 +0300 |
commit | 34f6765630c7d78f47020e9b915a62193d88ead2 (patch) | |
tree | 4d4c11467f7f7feed2b49503a3b6f41c38933b4b | |
parent | c58d1acba89824d02eb6939b40df7c92b4db3581 (diff) | |
parent | 721fad37a1981a564404b5f708a405503dc18f45 (diff) |
Merge branch 'master' into temp-multi-function-procedure
59 files changed, 2039 insertions, 1121 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 2868324bf46..5a555876d21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1598,6 +1598,10 @@ elseif(CMAKE_C_COMPILER_ID MATCHES "Clang") ADD_CHECK_C_COMPILER_FLAG(C_WARNINGS C_WARN_UNUSED_PARAMETER -Wunused-parameter) ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_ALL -Wall) + # Designated initializer is a C++20 feature & breaks MSVC build. Dropping MSVC 2019 or + # updating to C++20 allows removing this. + ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_CXX20_DESIGNATOR -Wc++20-designator) + ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_AUTOLOGICAL_COMPARE -Wno-tautological-compare) ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_UNKNOWN_PRAGMAS -Wno-unknown-pragmas) ADD_CHECK_CXX_COMPILER_FLAG(CXX_WARNINGS CXX_WARN_NO_CHAR_SUBSCRIPTS -Wno-char-subscripts) diff --git a/build_files/cmake/Modules/FindZstd.cmake b/build_files/cmake/Modules/FindZstd.cmake new file mode 100644 index 00000000000..84606d01f44 --- /dev/null +++ b/build_files/cmake/Modules/FindZstd.cmake @@ -0,0 +1,66 @@ +# - Find Zstd library +# Find the native Zstd includes and library +# This module defines +# ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when +# ZSTD_INCLUDE_DIR is found. +# ZSTD_LIBRARIES, libraries to link against to use Zstd. +# ZSTD_ROOT_DIR, The base directory to search for Zstd. +# This can also be an environment variable. +# ZSTD_FOUND, If false, do not try to use Zstd. +# +# also defined, but not for general use are +# ZSTD_LIBRARY, where to find the Zstd library. + +#============================================================================= +# Copyright 2019 Blender Foundation. +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= + +# If ZSTD_ROOT_DIR was defined in the environment, use it. +IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "") + SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR}) +ENDIF() + +SET(_zstd_SEARCH_DIRS + ${ZSTD_ROOT_DIR} +) + +FIND_PATH(ZSTD_INCLUDE_DIR + NAMES + zstd.h + HINTS + ${_zstd_SEARCH_DIRS} + PATH_SUFFIXES + include +) + +FIND_LIBRARY(ZSTD_LIBRARY + NAMES + zstd + HINTS + ${_zstd_SEARCH_DIRS} + PATH_SUFFIXES + lib64 lib + ) + +# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG + ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +IF(ZSTD_FOUND) + SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY}) + SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR}) +ENDIF() + +MARK_AS_ADVANCED( + ZSTD_INCLUDE_DIR + ZSTD_LIBRARY +) diff --git a/build_files/cmake/platform/platform_apple.cmake b/build_files/cmake/platform/platform_apple.cmake index dceafb236de..2bfdc84ec2a 100644 --- a/build_files/cmake/platform/platform_apple.cmake +++ b/build_files/cmake/platform/platform_apple.cmake @@ -441,6 +441,9 @@ if(WITH_HARU) endif() endif() +set(ZSTD_ROOT_DIR ${LIBDIR}/zstd) +find_package(Zstd REQUIRED) + if(EXISTS ${LIBDIR}) without_system_libs_end() endif() diff --git a/build_files/cmake/platform/platform_unix.cmake b/build_files/cmake/platform/platform_unix.cmake index 7f62399ac4f..fc0c37e4c8b 100644 --- a/build_files/cmake/platform/platform_unix.cmake +++ b/build_files/cmake/platform/platform_unix.cmake @@ -99,6 +99,7 @@ endif() find_package_wrapper(JPEG REQUIRED) find_package_wrapper(PNG REQUIRED) find_package_wrapper(ZLIB REQUIRED) +find_package_wrapper(Zstd REQUIRED) find_package_wrapper(Freetype REQUIRED) if(WITH_PYTHON) diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake index 3773aaaffed..d44ef691d1b 100644 --- a/build_files/cmake/platform/platform_win32.cmake +++ b/build_files/cmake/platform/platform_win32.cmake @@ -873,3 +873,6 @@ if(WITH_HARU) set(WITH_HARU OFF) endif() endif() + +set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include) +set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib) diff --git a/doc/python_api/requirements.txt b/doc/python_api/requirements.txt index a854d2a267b..b5a9d15bf7b 100644 --- a/doc/python_api/requirements.txt +++ b/doc/python_api/requirements.txt @@ -1,13 +1,13 @@ -sphinx==3.5.4 +sphinx==4.1.1 # Sphinx dependencies that are important -Jinja2==2.11.3 -Pygments==2.9.0 -docutils==0.16 +Jinja2==3.0.1 +Pygments==2.10.0 +docutils==0.17.1 snowballstemmer==2.1.0 babel==2.9.1 -requests==2.25.1 +requests==2.26.0 # Only needed to match the theme used for the official documentation. # Without this theme, the default theme will be used. -sphinx_rtd_theme==0.5.2 +sphinx_rtd_theme==1.0.0rc1 diff --git a/intern/cycles/blender/addon/properties.py b/intern/cycles/blender/addon/properties.py index ac0aca57028..0c3af3fabeb 100644 --- a/intern/cycles/blender/addon/properties.py +++ b/intern/cycles/blender/addon/properties.py @@ -408,7 +408,7 @@ class CyclesRenderSettings(bpy.types.PropertyGroup): adaptive_threshold: FloatProperty( name="Adaptive Sampling Threshold", - description="Noise level step to stop sampling at, lower values reduce noise the cost of render time. Zero for automatic setting based on number of AA samples", + description="Noise level step to stop sampling at, lower values reduce noise at the cost of render time. Zero for automatic setting based on number of AA samples", min=0.0, max=1.0, default=0.0, precision=4, diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp index 2dbebac4cc3..a7eae421b55 100644 --- a/intern/cycles/blender/blender_object.cpp +++ b/intern/cycles/blender/blender_object.cpp @@ -523,6 +523,9 @@ void BlenderSync::sync_procedural(BL::Object &b_ob, procedural->set_scale(cache_file.scale()); + procedural->set_use_prefetch(cache_file.use_prefetch()); + procedural->set_prefetch_cache_size(cache_file.prefetch_cache_size()); + /* create or update existing AlembicObjects */ ustring object_path = ustring(b_mesh_cache.object_path()); diff --git a/intern/cycles/render/alembic.cpp b/intern/cycles/render/alembic.cpp index 81f47256739..69bc0712674 100644 --- a/intern/cycles/render/alembic.cpp +++ b/intern/cycles/render/alembic.cpp @@ -25,6 +25,7 @@ #include "render/shader.h" #include "util/util_foreach.h" +#include "util/util_logging.h" #include "util/util_progress.h" #include "util/util_transform.h" #include "util/util_vector.h" @@ -211,6 +212,35 @@ void CachedData::set_time_sampling(TimeSampling time_sampling) } } +size_t CachedData::memory_used() const +{ + size_t mem_used = 0; + + mem_used += curve_first_key.memory_used(); + mem_used += curve_keys.memory_used(); + mem_used += curve_radius.memory_used(); + mem_used += curve_shader.memory_used(); + mem_used += num_ngons.memory_used(); + mem_used += shader.memory_used(); + mem_used += subd_creases_edge.memory_used(); + mem_used += subd_creases_weight.memory_used(); + mem_used += subd_face_corners.memory_used(); + mem_used += subd_num_corners.memory_used(); + mem_used += subd_ptex_offset.memory_used(); + mem_used += subd_smooth.memory_used(); + mem_used += subd_start_corner.memory_used(); + mem_used += transforms.memory_used(); + mem_used += triangles.memory_used(); + mem_used += uv_loops.memory_used(); + mem_used += vertices.memory_used(); + + for (const CachedAttribute &attr : attributes) { + mem_used += attr.data.memory_used(); + } + + return mem_used; +} + static M44d convert_yup_zup(const M44d &mtx, float scale_mult) { V3d scale, shear, rotation, translation; @@ -706,6 +736,9 @@ NODE_DEFINE(AlembicProcedural) SOCKET_NODE_ARRAY(objects, "Objects", AlembicObject::get_node_type()); + SOCKET_BOOLEAN(use_prefetch, "Use Prefetch", true); + SOCKET_INT(prefetch_cache_size, "Prefetch Cache Size", 4096); + return type; } @@ -823,6 +856,30 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress) } } + if (use_prefetch_is_modified()) { + if (!use_prefetch) { + for (Node *node : objects) { + AlembicObject *object = static_cast<AlembicObject *>(node); + object->clear_cache(); + } + } + } + + if (prefetch_cache_size_is_modified()) { + /* Check whether the current memory usage fits in the new requested size, + * abort the render if it is any higher. */ + size_t memory_used = 0ul; + for (Node *node : objects) { + AlembicObject *object = static_cast<AlembicObject *>(node); + memory_used += object->get_cached_data().memory_used(); + } + + if (memory_used > get_prefetch_cache_size_in_bytes()) { + progress.set_error("Error: Alembic Procedural memory limit reached"); + return; + } + } + build_caches(progress); foreach (Node *node, objects) { @@ -1300,6 +1357,8 @@ void AlembicProcedural::walk_hierarchy( void AlembicProcedural::build_caches(Progress &progress) { + size_t memory_used = 0; + for (Node *node : objects) { AlembicObject *object = static_cast<AlembicObject *>(node); @@ -1353,7 +1412,18 @@ void AlembicProcedural::build_caches(Progress &progress) if (scale_is_modified() || object->get_cached_data().transforms.size() == 0) { object->setup_transform_cache(object->get_cached_data(), scale); } + + memory_used += object->get_cached_data().memory_used(); + + if (use_prefetch) { + if (memory_used > get_prefetch_cache_size_in_bytes()) { + progress.set_error("Error: Alembic Procedural memory limit reached"); + return; + } + } } + + VLOG(1) << "AlembicProcedural memory usage : " << string_human_readable_size(memory_used); } CCL_NAMESPACE_END diff --git a/intern/cycles/render/alembic.h b/intern/cycles/render/alembic.h index 9c58af720f6..8e166a5ab04 100644 --- a/intern/cycles/render/alembic.h +++ b/intern/cycles/render/alembic.h @@ -272,6 +272,21 @@ template<typename T> class DataStore { node->set(*socket, value); } + size_t memory_used() const + { + if constexpr (is_array<T>::value) { + size_t mem_used = 0; + + for (const T &array : data) { + mem_used += array.size() * sizeof(array[0]); + } + + return mem_used; + } + + return data.size() * sizeof(T); + } + private: const TimeIndexPair &get_index_for_time(double time) const { @@ -332,6 +347,8 @@ struct CachedData { void invalidate_last_loaded_time(bool attributes_only = false); void set_time_sampling(Alembic::AbcCoreAbstract::TimeSampling time_sampling); + + size_t memory_used() const; }; /* Representation of an Alembic object for the AlembicProcedural. @@ -482,6 +499,13 @@ class AlembicProcedural : public Procedural { * software. */ NODE_SOCKET_API(float, scale) + /* Cache controls */ + NODE_SOCKET_API(bool, use_prefetch) + + /* Memory limit for the cache, if the data does not fit within this limit, rendering is aborted. + */ + NODE_SOCKET_API(int, prefetch_cache_size) + AlembicProcedural(); ~AlembicProcedural(); @@ -531,6 +555,12 @@ class AlembicProcedural : public Procedural { void read_subd(AlembicObject *abc_object, Alembic::AbcGeom::Abc::chrono_t frame_time); void build_caches(Progress &progress); + + size_t get_prefetch_cache_size_in_bytes() const + { + /* prefetch_cache_size is in megabytes, so convert to bytes. */ + return static_cast<size_t>(prefetch_cache_size) * 1024 * 1024; + } }; CCL_NAMESPACE_END diff --git a/intern/cycles/render/alembic_read.cpp b/intern/cycles/render/alembic_read.cpp index c53ec668938..b105af63b44 100644 --- a/intern/cycles/render/alembic_read.cpp +++ b/intern/cycles/render/alembic_read.cpp @@ -44,9 +44,19 @@ static set<chrono_t> get_relevant_sample_times(AlembicProcedural *proc, return result; } - // load the data for the entire animation - const double start_frame = static_cast<double>(proc->get_start_frame()); - const double end_frame = static_cast<double>(proc->get_end_frame()); + double start_frame; + double end_frame; + + if (proc->get_use_prefetch()) { + // load the data for the entire animation + start_frame = static_cast<double>(proc->get_start_frame()); + end_frame = static_cast<double>(proc->get_end_frame()); + } + else { + // load the data for the current frame + start_frame = static_cast<double>(proc->get_frame()); + end_frame = start_frame; + } const double frame_rate = static_cast<double>(proc->get_frame_rate()); const double start_time = start_frame / frame_rate; diff --git a/intern/ghost/GHOST_IEvent.h b/intern/ghost/GHOST_IEvent.h index bcccd536ebd..239eea21088 100644 --- a/intern/ghost/GHOST_IEvent.h +++ b/intern/ghost/GHOST_IEvent.h @@ -32,10 +32,10 @@ class GHOST_IWindow; /** * Interface class for events received from GHOST. * You should not need to inherit this class. The system will pass these events - * to the GHOST_IEventConsumer::processEvent() method of event consumers.<br> - * Use the getType() method to retrieve the type of event and the getData() + * to the #GHOST_IEventConsumer::processEvent() method of event consumers.<br> + * Use the #getType() method to retrieve the type of event and the #getData() * method to get the event data out. Using the event type you can cast the - * event data to the correct event dat structure. + * event data to the correct event data structure. * \see GHOST_IEventConsumer#processEvent * \see GHOST_TEventType */ diff --git a/intern/ghost/intern/GHOST_SystemWin32.cpp b/intern/ghost/intern/GHOST_SystemWin32.cpp index 347067eae50..f44107ee000 100644 --- a/intern/ghost/intern/GHOST_SystemWin32.cpp +++ b/intern/ghost/intern/GHOST_SystemWin32.cpp @@ -1741,7 +1741,7 @@ LRESULT WINAPI GHOST_SystemWin32::s_wndProc(HWND hwnd, UINT msg, WPARAM wParam, case WM_MOUSELEAVE: { window->m_mousePresent = false; if (window->getTabletData().Active == GHOST_kTabletModeNone) { - processCursorEvent(window); + event = processCursorEvent(window); } GHOST_Wintab *wt = window->getWintab(); if (wt) { diff --git a/source/blender/blenfont/intern/blf.c b/source/blender/blenfont/intern/blf.c index 9168e7aa19c..2f9eb0753ac 100644 --- a/source/blender/blenfont/intern/blf.c +++ b/source/blender/blenfont/intern/blf.c @@ -108,7 +108,6 @@ void BLF_cache_clear(void) FontBLF *font = global_font[i]; if (font) { blf_glyph_cache_clear(font); - blf_kerning_cache_clear(font); } } } diff --git a/source/blender/blenfont/intern/blf_font.c b/source/blender/blenfont/intern/blf_font.c index 00d3cfb09eb..75a2e893119 100644 --- a/source/blender/blenfont/intern/blf_font.c +++ b/source/blender/blenfont/intern/blf_font.c @@ -72,6 +72,38 @@ static SpinLock ft_lib_mutex; static SpinLock blf_glyph_cache_mutex; /* -------------------------------------------------------------------- */ +/** \name FreeType Utilities (Internal) + * \{ */ + +/** + * Convert a FreeType 26.6 value representing an unscaled design size to pixels. + * This is an exact copy of the scaling done inside FT_Get_Kerning when called + * with #FT_KERNING_DEFAULT, including arbitrary resizing for small fonts. + */ +static int blf_unscaled_F26Dot6_to_pixels(FontBLF *font, FT_Pos value) +{ + /* Scale value by font size using integer-optimized multiplication. */ + FT_Long scaled = FT_MulFix(value, font->face->size->metrics.x_scale); + + /* FreeType states that this '25' has been determined heuristically. */ + if (font->face->size->metrics.x_ppem < 25) { + scaled = FT_MulDiv(scaled, font->face->size->metrics.x_ppem, 25); + } + + /* Copies of internal FreeType macros needed here. */ +#define FT_PIX_FLOOR(x) ((x) & ~63) +#define FT_PIX_ROUND(x) FT_PIX_FLOOR((x) + 32) + + /* Round to even 64ths, then divide by 64. */ + return (int)FT_PIX_ROUND(scaled) >> 6; + +#undef FT_PIX_FLOOR +#undef FT_PIX_ROUND +} + +/** \} */ + +/* -------------------------------------------------------------------- */ /** \name Glyph Batching * \{ */ @@ -257,67 +289,8 @@ static void blf_batch_draw_end(void) /** \} */ /* -------------------------------------------------------------------- */ - -int blf_font_init(void) -{ - memset(&g_batch, 0, sizeof(g_batch)); - BLI_spin_init(&ft_lib_mutex); - BLI_spin_init(&blf_glyph_cache_mutex); - return FT_Init_FreeType(&ft_lib); -} - -void blf_font_exit(void) -{ - FT_Done_FreeType(ft_lib); - BLI_spin_end(&ft_lib_mutex); - BLI_spin_end(&blf_glyph_cache_mutex); - blf_batch_draw_exit(); -} - -void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi) -{ - GlyphCacheBLF *gc; - FT_Error err; - - blf_glyph_cache_acquire(font); - - gc = blf_glyph_cache_find(font, size, dpi); - if (gc) { - /* Optimization: do not call FT_Set_Char_Size if size did not change. */ - if (font->size == size && font->dpi == dpi) { - blf_glyph_cache_release(font); - return; - } - } - - err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi); - if (err) { - /* FIXME: here we can go through the fixed size and choice a close one */ - printf("The current font don't support the size, %u and dpi, %u\n", size, dpi); - - blf_glyph_cache_release(font); - return; - } - - font->size = size; - font->dpi = dpi; - - if (!gc) { - blf_glyph_cache_new(font); - } - blf_glyph_cache_release(font); -} - -static void blf_font_ensure_ascii_kerning(FontBLF *font, GlyphCacheBLF *gc) -{ - if (font->kerning_cache || !FT_HAS_KERNING(font->face)) { - return; - } - font->kerning_cache = blf_kerning_cache_find(font); - if (!font->kerning_cache) { - font->kerning_cache = blf_kerning_cache_new(font, gc); - } -} +/** \name Glyph Stepping Utilities (Internal) + * \{ */ /* Fast path for runs of ASCII characters. Given that common UTF-8 * input will consist of an overwhelming majority of ASCII @@ -355,22 +328,40 @@ BLI_INLINE void blf_kerning_step_fast(FontBLF *font, const uint c, int *pen_x_p) { - /* `blf_font_ensure_ascii_kerning(font, gc);` must be called before this function. */ - BLI_assert(font->kerning_cache != NULL || !FT_HAS_KERNING(font->face)); + if (!FT_HAS_KERNING(font->face) || g_prev == NULL) { + return; + } - if (g_prev != NULL && FT_HAS_KERNING(font->face)) { - if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) { - *pen_x_p += font->kerning_cache->ascii_table[c][c_prev]; - } - else { - FT_Vector delta; - if (FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_DEFAULT, &delta) == 0) { - *pen_x_p += (int)delta.x >> 6; - } - } + FT_Vector delta = {KERNING_ENTRY_UNSET}; + + /* Get unscaled kerning value from our cache if ASCII. */ + if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) { + delta.x = font->kerning_cache->ascii_table[c][c_prev]; + } + + /* If not ASCII or not found in cache, ask FreeType for kerning. */ + if (UNLIKELY(delta.x == KERNING_ENTRY_UNSET)) { + /* Note that this function sets delta values to zero on any error. */ + FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_UNSCALED, &delta); + } + + /* If ASCII we save this value to our cache for quicker access next time. */ + if ((c_prev < KERNING_CACHE_TABLE_SIZE) && (c < GLYPH_ASCII_TABLE_SIZE)) { + font->kerning_cache->ascii_table[c][c_prev] = (int)delta.x; + } + + if (delta.x != 0) { + /* Convert unscaled design units to pixels and move pen. */ + *pen_x_p += blf_unscaled_F26Dot6_to_pixels(font, delta.x); } } +/** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Text Drawing: GPU + * \{ */ + static void blf_font_draw_ex(FontBLF *font, GlyphCacheBLF *gc, const char *str, @@ -388,8 +379,6 @@ static void blf_font_draw_ex(FontBLF *font, return; } - blf_font_ensure_ascii_kerning(font, gc); - blf_batch_draw_begin(font); while ((i < len) && str[i]) { @@ -435,8 +424,6 @@ static void blf_font_draw_ascii_ex( GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - blf_font_ensure_ascii_kerning(font, gc); - blf_batch_draw_begin(font); while ((c = *(str++)) && len--) { @@ -515,6 +502,12 @@ int blf_font_draw_mono(FontBLF *font, const char *str, size_t len, int cwidth) return columns; } +/** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Text Drawgin: Buffer + * \{ */ + /* Sanity checks are done by BLF_draw_buffer() */ static void blf_font_draw_buffer_ex(FontBLF *font, GlyphCacheBLF *gc, @@ -536,8 +529,6 @@ static void blf_font_draw_buffer_ex(FontBLF *font, int chx, chy; int y, x; - blf_font_ensure_ascii_kerning(font, gc); - /* another buffer specific call for color conversion */ while ((i < len) && str[i]) { @@ -662,6 +653,16 @@ void blf_font_draw_buffer(FontBLF *font, const char *str, size_t len, struct Res blf_glyph_cache_release(font); } +/** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Text Evaluation: Width to Sting Length + * + * Use to implement exported functions: + * - #BLF_width_to_strlen + * - #BLF_width_to_rstrlen + * \{ */ + static bool blf_font_width_to_strlen_glyph_process(FontBLF *font, const uint c_prev, const uint c, @@ -694,8 +695,6 @@ size_t blf_font_width_to_strlen( GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); const int width_i = (int)width; - blf_font_ensure_ascii_kerning(font, gc); - for (i_prev = i = 0, width_new = pen_x = 0, g_prev = NULL, c_prev = 0; (i < len) && str[i]; i_prev = i, width_new = pen_x, c_prev = c, g_prev = g) { g = blf_utf8_next_fast(font, gc, str, &i, &c); @@ -725,8 +724,6 @@ size_t blf_font_width_to_rstrlen( GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); const int width_i = (int)width; - blf_font_ensure_ascii_kerning(font, gc); - i = BLI_strnlen(str, len); s = BLI_str_find_prev_char_utf8(str, &str[i]); i = (size_t)((s != NULL) ? s - str : 0); @@ -759,6 +756,12 @@ size_t blf_font_width_to_rstrlen( return i; } +/** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Text Evaluation: Glyph Bound Box with Callback + * \{ */ + static void blf_font_boundbox_ex(FontBLF *font, GlyphCacheBLF *gc, const char *str, @@ -778,8 +781,6 @@ static void blf_font_boundbox_ex(FontBLF *font, box->ymin = 32000.0f; box->ymax = -32000.0f; - blf_font_ensure_ascii_kerning(font, gc); - while ((i < len) && str[i]) { g = blf_utf8_next_fast(font, gc, str, &i, &c); @@ -835,8 +836,165 @@ void blf_font_boundbox( blf_glyph_cache_release(font); } +void blf_font_width_and_height(FontBLF *font, + const char *str, + size_t len, + float *r_width, + float *r_height, + struct ResultBLF *r_info) +{ + float xa, ya; + rctf box; + + if (font->flags & BLF_ASPECT) { + xa = font->aspect[0]; + ya = font->aspect[1]; + } + else { + xa = 1.0f; + ya = 1.0f; + } + + if (font->flags & BLF_WORD_WRAP) { + blf_font_boundbox__wrap(font, str, len, &box, r_info); + } + else { + blf_font_boundbox(font, str, len, &box, r_info); + } + *r_width = (BLI_rctf_size_x(&box) * xa); + *r_height = (BLI_rctf_size_y(&box) * ya); +} + +float blf_font_width(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info) +{ + float xa; + rctf box; + + if (font->flags & BLF_ASPECT) { + xa = font->aspect[0]; + } + else { + xa = 1.0f; + } + + if (font->flags & BLF_WORD_WRAP) { + blf_font_boundbox__wrap(font, str, len, &box, r_info); + } + else { + blf_font_boundbox(font, str, len, &box, r_info); + } + return BLI_rctf_size_x(&box) * xa; +} + +float blf_font_height(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info) +{ + float ya; + rctf box; + + if (font->flags & BLF_ASPECT) { + ya = font->aspect[1]; + } + else { + ya = 1.0f; + } + + if (font->flags & BLF_WORD_WRAP) { + blf_font_boundbox__wrap(font, str, len, &box, r_info); + } + else { + blf_font_boundbox(font, str, len, &box, r_info); + } + return BLI_rctf_size_y(&box) * ya; +} + +float blf_font_fixed_width(FontBLF *font) +{ + const unsigned int c = ' '; + + GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); + GlyphBLF *g = blf_glyph_search(gc, c); + if (!g) { + g = blf_glyph_add(font, gc, FT_Get_Char_Index(font->face, c), c); + + /* if we don't find the glyph. */ + if (!g) { + blf_glyph_cache_release(font); + return 0.0f; + } + } + + blf_glyph_cache_release(font); + return g->advance; +} + +static void blf_font_boundbox_foreach_glyph_ex(FontBLF *font, + GlyphCacheBLF *gc, + const char *str, + size_t len, + BLF_GlyphBoundsFn user_fn, + void *user_data, + struct ResultBLF *r_info, + int pen_y) +{ + unsigned int c, c_prev = BLI_UTF8_ERR; + GlyphBLF *g, *g_prev = NULL; + int pen_x = 0; + size_t i = 0, i_curr; + rcti gbox; + + if (len == 0) { + /* early output. */ + return; + } + + while ((i < len) && str[i]) { + i_curr = i; + g = blf_utf8_next_fast(font, gc, str, &i, &c); + + if (UNLIKELY(c == BLI_UTF8_ERR)) { + break; + } + if (UNLIKELY(g == NULL)) { + continue; + } + blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x); + + gbox.xmin = pen_x; + gbox.xmax = gbox.xmin + MIN2(g->advance_i, g->dims[0]); + gbox.ymin = pen_y; + gbox.ymax = gbox.ymin - g->dims[1]; + + pen_x += g->advance_i; + + if (user_fn(str, i_curr, &gbox, g->advance_i, &g->box, g->pos, user_data) == false) { + break; + } + + g_prev = g; + c_prev = c; + } + + if (r_info) { + r_info->lines = 1; + r_info->width = pen_x; + } +} +void blf_font_boundbox_foreach_glyph(FontBLF *font, + const char *str, + size_t len, + BLF_GlyphBoundsFn user_fn, + void *user_data, + struct ResultBLF *r_info) +{ + GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); + blf_font_boundbox_foreach_glyph_ex(font, gc, str, len, user_fn, user_data, r_info, 0); + blf_glyph_cache_release(font); +} + +/** \} */ + /* -------------------------------------------------------------------- */ -/** \name Word-Wrap Support +/** \name Text Evaluation: Word-Wrap with Callback * \{ */ /** @@ -869,8 +1027,6 @@ static void blf_font_wrap_apply(FontBLF *font, GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - blf_font_ensure_ascii_kerning(font, gc); - struct WordWrapVars { int wrap_width; size_t start, last[2]; @@ -1008,216 +1164,120 @@ void blf_font_draw_buffer__wrap(FontBLF *font, /** \} */ -void blf_font_width_and_height(FontBLF *font, - const char *str, - size_t len, - float *r_width, - float *r_height, - struct ResultBLF *r_info) +/* -------------------------------------------------------------------- */ +/** \name Text Evaluation: Count Missing Characters + * \{ */ + +int blf_font_count_missing_chars(FontBLF *font, + const char *str, + const size_t len, + int *r_tot_chars) { - float xa, ya; - rctf box; + int missing = 0; + size_t i = 0; - if (font->flags & BLF_ASPECT) { - xa = font->aspect[0]; - ya = font->aspect[1]; - } - else { - xa = 1.0f; - ya = 1.0f; - } + *r_tot_chars = 0; + while (i < len) { + unsigned int c; - if (font->flags & BLF_WORD_WRAP) { - blf_font_boundbox__wrap(font, str, len, &box, r_info); - } - else { - blf_font_boundbox(font, str, len, &box, r_info); + if ((c = str[i]) < GLYPH_ASCII_TABLE_SIZE) { + i++; + } + else if ((c = BLI_str_utf8_as_unicode_step(str, &i)) != BLI_UTF8_ERR) { + if (FT_Get_Char_Index((font)->face, c) == 0) { + missing++; + } + } + (*r_tot_chars)++; } - *r_width = (BLI_rctf_size_x(&box) * xa); - *r_height = (BLI_rctf_size_y(&box) * ya); + return missing; } -float blf_font_width(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info) -{ - float xa; - rctf box; - - if (font->flags & BLF_ASPECT) { - xa = font->aspect[0]; - } - else { - xa = 1.0f; - } +/** \} */ - if (font->flags & BLF_WORD_WRAP) { - blf_font_boundbox__wrap(font, str, len, &box, r_info); - } - else { - blf_font_boundbox(font, str, len, &box, r_info); - } - return BLI_rctf_size_x(&box) * xa; -} +/* -------------------------------------------------------------------- */ +/** \name Font Query: Attributes + * \{ */ -float blf_font_height(FontBLF *font, const char *str, size_t len, struct ResultBLF *r_info) +int blf_font_height_max(FontBLF *font) { - float ya; - rctf box; + int height_max; - if (font->flags & BLF_ASPECT) { - ya = font->aspect[1]; - } - else { - ya = 1.0f; - } + GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); + height_max = gc->glyph_height_max; - if (font->flags & BLF_WORD_WRAP) { - blf_font_boundbox__wrap(font, str, len, &box, r_info); - } - else { - blf_font_boundbox(font, str, len, &box, r_info); - } - return BLI_rctf_size_y(&box) * ya; + blf_glyph_cache_release(font); + return height_max; } -float blf_font_fixed_width(FontBLF *font) +int blf_font_width_max(FontBLF *font) { - const unsigned int c = ' '; + int width_max; GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - GlyphBLF *g = blf_glyph_search(gc, c); - if (!g) { - g = blf_glyph_add(font, gc, FT_Get_Char_Index(font->face, c), c); - - /* if we don't find the glyph. */ - if (!g) { - blf_glyph_cache_release(font); - return 0.0f; - } - } + width_max = gc->glyph_width_max; blf_glyph_cache_release(font); - return g->advance; + return width_max; } -/* -------------------------------------------------------------------- */ -/** \name Glyph Bound Box with Callback - * \{ */ - -static void blf_font_boundbox_foreach_glyph_ex(FontBLF *font, - GlyphCacheBLF *gc, - const char *str, - size_t len, - BLF_GlyphBoundsFn user_fn, - void *user_data, - struct ResultBLF *r_info, - int pen_y) +float blf_font_descender(FontBLF *font) { - unsigned int c, c_prev = BLI_UTF8_ERR; - GlyphBLF *g, *g_prev = NULL; - int pen_x = 0; - size_t i = 0, i_curr; - rcti gbox; - - if (len == 0) { - /* early output. */ - return; - } - - blf_font_ensure_ascii_kerning(font, gc); - - while ((i < len) && str[i]) { - i_curr = i; - g = blf_utf8_next_fast(font, gc, str, &i, &c); - - if (UNLIKELY(c == BLI_UTF8_ERR)) { - break; - } - if (UNLIKELY(g == NULL)) { - continue; - } - blf_kerning_step_fast(font, g_prev, g, c_prev, c, &pen_x); + float descender; - gbox.xmin = pen_x; - gbox.xmax = gbox.xmin + MIN2(g->advance_i, g->dims[0]); - gbox.ymin = pen_y; - gbox.ymax = gbox.ymin - g->dims[1]; + GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); + descender = gc->descender; - pen_x += g->advance_i; + blf_glyph_cache_release(font); + return descender; +} - if (user_fn(str, i_curr, &gbox, g->advance_i, &g->box, g->pos, user_data) == false) { - break; - } +float blf_font_ascender(FontBLF *font) +{ + float ascender; - g_prev = g; - c_prev = c; - } + GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); + ascender = gc->ascender; - if (r_info) { - r_info->lines = 1; - r_info->width = pen_x; - } + blf_glyph_cache_release(font); + return ascender; } -void blf_font_boundbox_foreach_glyph(FontBLF *font, - const char *str, - size_t len, - BLF_GlyphBoundsFn user_fn, - void *user_data, - struct ResultBLF *r_info) + +char *blf_display_name(FontBLF *font) { - GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - blf_font_boundbox_foreach_glyph_ex(font, gc, str, len, user_fn, user_data, r_info, 0); - blf_glyph_cache_release(font); + if (!font->face->family_name) { + return NULL; + } + return BLI_sprintfN("%s %s", font->face->family_name, font->face->style_name); } /** \} */ -int blf_font_count_missing_chars(FontBLF *font, - const char *str, - const size_t len, - int *r_tot_chars) -{ - int missing = 0; - size_t i = 0; - - *r_tot_chars = 0; - while (i < len) { - unsigned int c; +/* -------------------------------------------------------------------- */ +/** \name Font Subsystem Init/Exit + * \{ */ - if ((c = str[i]) < GLYPH_ASCII_TABLE_SIZE) { - i++; - } - else if ((c = BLI_str_utf8_as_unicode_step(str, &i)) != BLI_UTF8_ERR) { - if (FT_Get_Char_Index((font)->face, c) == 0) { - missing++; - } - } - (*r_tot_chars)++; - } - return missing; +int blf_font_init(void) +{ + memset(&g_batch, 0, sizeof(g_batch)); + BLI_spin_init(&ft_lib_mutex); + BLI_spin_init(&blf_glyph_cache_mutex); + return FT_Init_FreeType(&ft_lib); } -void blf_font_free(FontBLF *font) +void blf_font_exit(void) { - BLI_spin_lock(&blf_glyph_cache_mutex); - GlyphCacheBLF *gc; - - while ((gc = BLI_pophead(&font->cache))) { - blf_glyph_cache_free(gc); - } - - blf_kerning_cache_clear(font); + FT_Done_FreeType(ft_lib); + BLI_spin_end(&ft_lib_mutex); + BLI_spin_end(&blf_glyph_cache_mutex); + blf_batch_draw_exit(); +} - FT_Done_Face(font->face); - if (font->filename) { - MEM_freeN(font->filename); - } - if (font->name) { - MEM_freeN(font->name); - } - MEM_freeN(font); +/** \} */ - BLI_spin_unlock(&blf_glyph_cache_mutex); -} +/* -------------------------------------------------------------------- */ +/** \name Font New/Free + * \{ */ static void blf_font_fill(FontBLF *font) { @@ -1246,7 +1306,6 @@ static void blf_font_fill(FontBLF *font) font->dpi = 0; font->size = 0; BLI_listbase_clear(&font->cache); - BLI_listbase_clear(&font->kerning_caches); font->kerning_cache = NULL; #if BLF_BLUR_ENABLE font->blur = 0; @@ -1307,6 +1366,17 @@ FontBLF *blf_font_new(const char *name, const char *filename) font->name = BLI_strdup(name); font->filename = BLI_strdup(filename); blf_font_fill(font); + + if (FT_HAS_KERNING(font->face)) { + /* Create kerning cache table and fill with value indicating "unset". */ + font->kerning_cache = MEM_mallocN(sizeof(KerningCacheBLF), __func__); + for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) { + for (uint j = 0; j < KERNING_CACHE_TABLE_SIZE; j++) { + font->kerning_cache->ascii_table[i][j] = KERNING_ENTRY_UNSET; + } + } + } + return font; } @@ -1346,54 +1416,61 @@ FontBLF *blf_font_new_from_mem(const char *name, const unsigned char *mem, int m return font; } -int blf_font_height_max(FontBLF *font) +void blf_font_free(FontBLF *font) { - int height_max; - - GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - height_max = gc->glyph_height_max; + BLI_spin_lock(&blf_glyph_cache_mutex); + GlyphCacheBLF *gc; - blf_glyph_cache_release(font); - return height_max; -} + while ((gc = BLI_pophead(&font->cache))) { + blf_glyph_cache_free(gc); + } -int blf_font_width_max(FontBLF *font) -{ - int width_max; + if (font->kerning_cache) { + MEM_freeN(font->kerning_cache); + } - GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - width_max = gc->glyph_width_max; + FT_Done_Face(font->face); + if (font->filename) { + MEM_freeN(font->filename); + } + if (font->name) { + MEM_freeN(font->name); + } + MEM_freeN(font); - blf_glyph_cache_release(font); - return width_max; + BLI_spin_unlock(&blf_glyph_cache_mutex); } -float blf_font_descender(FontBLF *font) -{ - float descender; - - GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - descender = gc->descender; +/** \} */ - blf_glyph_cache_release(font); - return descender; -} +/* -------------------------------------------------------------------- */ +/** \name Font Configure + * \{ */ -float blf_font_ascender(FontBLF *font) +void blf_font_size(FontBLF *font, unsigned int size, unsigned int dpi) { - float ascender; + blf_glyph_cache_acquire(font); - GlyphCacheBLF *gc = blf_glyph_cache_acquire(font); - ascender = gc->ascender; + GlyphCacheBLF *gc = blf_glyph_cache_find(font, size, dpi); + if (gc && (font->size == size && font->dpi == dpi)) { + /* Optimization: do not call FT_Set_Char_Size if size did not change. */ + } + else { + const FT_Error err = FT_Set_Char_Size(font->face, 0, ((FT_F26Dot6)(size)) * 64, dpi, dpi); + if (err) { + /* FIXME: here we can go through the fixed size and choice a close one */ + printf("The current font don't support the size, %u and dpi, %u\n", size, dpi); + } + else { + font->size = size; + font->dpi = dpi; + if (gc == NULL) { + blf_glyph_cache_new(font); + } + } + } blf_glyph_cache_release(font); - return ascender; } -char *blf_display_name(FontBLF *font) -{ - if (!font->face->family_name) { - return NULL; - } - return BLI_sprintfN("%s %s", font->face->family_name, font->face->style_name); -} +/** \} */ diff --git a/source/blender/blenfont/intern/blf_glyph.c b/source/blender/blenfont/intern/blf_glyph.c index 5fb69251466..6cdf5fc5996 100644 --- a/source/blender/blenfont/intern/blf_glyph.c +++ b/source/blender/blenfont/intern/blf_glyph.c @@ -55,56 +55,6 @@ #include "BLI_math_vector.h" #include "BLI_strict_flags.h" -KerningCacheBLF *blf_kerning_cache_find(FontBLF *font) -{ - return (KerningCacheBLF *)font->kerning_caches.first; -} - -/* Create a new glyph cache for the current kerning mode. */ -KerningCacheBLF *blf_kerning_cache_new(FontBLF *font, GlyphCacheBLF *gc) -{ - KerningCacheBLF *kc = MEM_mallocN(sizeof(KerningCacheBLF), __func__); - kc->next = NULL; - kc->prev = NULL; - - GlyphBLF *g_table[KERNING_CACHE_TABLE_SIZE]; - for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) { - GlyphBLF *g = blf_glyph_search(gc, i); - if (UNLIKELY(g == NULL)) { - FT_UInt glyph_index = FT_Get_Char_Index(font->face, i); - g = blf_glyph_add(font, gc, glyph_index, i); - } - g_table[i] = g; - } - - memset(kc->ascii_table, 0, sizeof(kc->ascii_table)); - for (uint i = 0; i < KERNING_CACHE_TABLE_SIZE; i++) { - GlyphBLF *g = g_table[i]; - if (g == NULL) { - continue; - } - for (uint j = 0; j < KERNING_CACHE_TABLE_SIZE; j++) { - GlyphBLF *g_prev = g_table[j]; - if (g_prev == NULL) { - continue; - } - FT_Vector delta; - if (FT_Get_Kerning(font->face, g_prev->idx, g->idx, FT_KERNING_DEFAULT, &delta) == 0) { - kc->ascii_table[i][j] = (int)delta.x >> 6; - } - } - } - - BLI_addhead(&font->kerning_caches, kc); - return kc; -} - -void blf_kerning_cache_clear(FontBLF *font) -{ - font->kerning_cache = NULL; - BLI_freelistN(&font->kerning_caches); -} - GlyphCacheBLF *blf_glyph_cache_find(FontBLF *font, unsigned int size, unsigned int dpi) { GlyphCacheBLF *p; diff --git a/source/blender/blenfont/intern/blf_internal.h b/source/blender/blenfont/intern/blf_internal.h index 35a6d019eac..ab2a26b1e06 100644 --- a/source/blender/blenfont/intern/blf_internal.h +++ b/source/blender/blenfont/intern/blf_internal.h @@ -121,10 +121,6 @@ int blf_font_count_missing_chars(struct FontBLF *font, void blf_font_free(struct FontBLF *font); -struct KerningCacheBLF *blf_kerning_cache_find(struct FontBLF *font); -struct KerningCacheBLF *blf_kerning_cache_new(struct FontBLF *font, struct GlyphCacheBLF *gc); -void blf_kerning_cache_clear(struct FontBLF *font); - struct GlyphCacheBLF *blf_glyph_cache_find(struct FontBLF *font, unsigned int size, unsigned int dpi); diff --git a/source/blender/blenfont/intern/blf_internal_types.h b/source/blender/blenfont/intern/blf_internal_types.h index caa10b2b125..38d7d7b6e21 100644 --- a/source/blender/blenfont/intern/blf_internal_types.h +++ b/source/blender/blenfont/intern/blf_internal_types.h @@ -34,6 +34,9 @@ /* Number of characters in KerningCacheBLF.table. */ #define KERNING_CACHE_TABLE_SIZE 128 +/* A value in the kerning cache that indicates it is not yet set. */ +#define KERNING_ENTRY_UNSET INT_MAX + typedef struct BatchBLF { struct FontBLF *font; /* can only batch glyph from the same font */ struct GPUBatch *batch; @@ -50,7 +53,6 @@ typedef struct BatchBLF { extern BatchBLF g_batch; typedef struct KerningCacheBLF { - struct KerningCacheBLF *next, *prev; /** * Cache a ascii glyph pairs. Only store the x offset we are interested in, * instead of the full #FT_Vector since it's not used for drawing at the moment. @@ -223,10 +225,7 @@ typedef struct FontBLF { */ ListBase cache; - /* list of kerning cache for this font. */ - ListBase kerning_caches; - - /* current kerning cache for this font and kerning mode. */ + /* Cache of unscaled kerning values. Will be NULL if font does not have kerning. */ KerningCacheBLF *kerning_cache; /* freetype2 lib handle. */ diff --git a/source/blender/blenkernel/BKE_cachefile.h b/source/blender/blenkernel/BKE_cachefile.h index 58d876b184b..836597f95da 100644 --- a/source/blender/blenkernel/BKE_cachefile.h +++ b/source/blender/blenkernel/BKE_cachefile.h @@ -61,11 +61,6 @@ void BKE_cachefile_reader_open(struct CacheFile *cache_file, const char *object_path); void BKE_cachefile_reader_free(struct CacheFile *cache_file, struct CacheReader **reader); -/* Determine whether the CacheFile should use a render engine procedural. If so, data is not read - * from the file and bouding boxes are used to represent the objects in the Scene. Render engines - * will receive the bounding box as a placeholder but can instead load the data directly if they - * support it. - */ bool BKE_cache_file_uses_render_procedural(const struct CacheFile *cache_file, struct Scene *scene, const int dag_eval_mode); diff --git a/source/blender/blenkernel/intern/cachefile.c b/source/blender/blenkernel/intern/cachefile.c index 4a60564b4a1..87b1584d422 100644 --- a/source/blender/blenkernel/intern/cachefile.c +++ b/source/blender/blenkernel/intern/cachefile.c @@ -368,7 +368,7 @@ void BKE_cachefile_eval(Main *bmain, Depsgraph *depsgraph, CacheFile *cache_file #endif if (DEG_is_active(depsgraph)) { - /* Flush object paths back to original datablock for UI. */ + /* Flush object paths back to original data-block for UI. */ CacheFile *cache_file_orig = (CacheFile *)DEG_get_original_id(&cache_file->id); BLI_freelistN(&cache_file_orig->object_paths); BLI_duplicatelist(&cache_file_orig->object_paths, &cache_file->object_paths); @@ -411,6 +411,12 @@ float BKE_cachefile_time_offset(const CacheFile *cache_file, const float time, c return cache_file->is_sequence ? frame : frame / fps - time_offset; } +/** + * Determine whether the #CacheFile should use a render engine procedural. If so, data is not read + * from the file and bounding boxes are used to represent the objects in the Scene. + * Render engines will receive the bounding box as a placeholder but can instead + * load the data directly if they support it. + */ bool BKE_cache_file_uses_render_procedural(const CacheFile *cache_file, Scene *scene, const int dag_eval_mode) diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h index 7cfecc798a7..906a56ce909 100644 --- a/source/blender/blenlib/BLI_fileops.h +++ b/source/blender/blenlib/BLI_fileops.h @@ -154,18 +154,17 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL bool BLI_file_touch(const char *file) ATTR_NONNULL(); bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT; -#if 0 /* UNUSED */ -int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); -#endif -char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT - ATTR_NONNULL(); -size_t BLI_gzip_mem_to_file_at_pos(void *buf, - size_t len, - FILE *file, - size_t gz_stream_offset, - int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); -size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) +bool BLI_file_magic_is_gzip(const char header[4]); + +size_t BLI_file_zstd_from_mem_at_pos(void *buf, + size_t len, + FILE *file, + size_t file_offset, + int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +bool BLI_file_magic_is_zstd(const char header[4]); + size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT; size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h new file mode 100644 index 00000000000..8d1fa3d1596 --- /dev/null +++ b/source/blender/blenlib/BLI_filereader.h @@ -0,0 +1,81 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV. + * All rights reserved. + */ + +/** \file + * \ingroup bli + * \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...). + */ + +#pragma once + +#ifdef WIN32 +# include "BLI_winstuff.h" +#else +# include <sys/types.h> +#endif + +#include "BLI_compiler_attrs.h" +#include "BLI_utildefines.h" + +#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__) +typedef int64_t off64_t; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct FileReader; + +typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size); +typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence); +typedef void (*FileReaderCloseFn)(struct FileReader *reader); + +/* General structure for all FileReaders, implementations add custom fields at the end. */ +typedef struct FileReader { + FileReaderReadFn read; + FileReaderSeekFn seek; + FileReaderCloseFn close; + + off64_t offset; +} FileReader; + +/* Functions for opening the various types of FileReader. + * They either succeed and return a valid FileReader, or fail and return NULL. + * + * If a FileReader is created, it has to be cleaned up and freed by calling + * its close() function unless another FileReader has taken ownership - for example, + * Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called. + */ + +/* Create FileReader from raw file descriptor. */ +FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT; +/* Create FileReader from raw file descriptor using memory-mapped IO. */ +FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT; +/* Create FileReader from a region of memory. */ +FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT + ATTR_NONNULL(); +/* Create FileReader from applying Zstd decompression on an underlying file. */ +FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +/* Create FileReader from applying Gzip decompression on an underlying file. */ +FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); + +#ifdef __cplusplus +} +#endif diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt index b6603dce378..f98d15ad08b 100644 --- a/source/blender/blenlib/CMakeLists.txt +++ b/source/blender/blenlib/CMakeLists.txt @@ -31,6 +31,7 @@ set(INC set(INC_SYS ${ZLIB_INCLUDE_DIRS} + ${ZSTD_INCLUDE_DIRS} ${FREETYPE_INCLUDE_DIRS} ${GMP_INCLUDE_DIRS} ) @@ -75,6 +76,10 @@ set(SRC intern/endian_switch.c intern/expr_pylike_eval.c intern/fileops.c + intern/filereader_file.c + intern/filereader_gzip.c + intern/filereader_memory.c + intern/filereader_zstd.c intern/fnmatch.c intern/freetypefont.c intern/gsqueue.c @@ -194,6 +199,7 @@ set(SRC BLI_enumerable_thread_specific.hh BLI_expr_pylike_eval.h BLI_fileops.h + BLI_filereader.h BLI_fileops_types.h BLI_float2.hh BLI_float3.hh @@ -323,6 +329,7 @@ set(LIB ${FREETYPE_LIBRARY} ${ZLIB_LIBRARIES} + ${ZSTD_LIBRARIES} ) if(WITH_MEM_VALGRIND) diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c index ac034d2b5cd..532a29b8147 100644 --- a/source/blender/blenlib/intern/fileops.c +++ b/source/blender/blenlib/intern/fileops.c @@ -31,6 +31,7 @@ #include <errno.h> #include "zlib.h" +#include "zstd.h" #ifdef WIN32 # include "BLI_fileops_types.h" @@ -61,200 +62,124 @@ #include "BLI_sys_types.h" /* for intptr_t support */ #include "BLI_utildefines.h" -#if 0 /* UNUSED */ -/* gzip the file in from and write it to "to". - * return -1 if zlib fails, -2 if the originating file does not exist - * NOTE: will remove the "from" file - */ -int BLI_file_gzip(const char *from, const char *to) +size_t BLI_file_zstd_from_mem_at_pos( + void *buf, size_t len, FILE *file, size_t file_offset, int compression_level) { - char buffer[10240]; - int file; - int readsize = 0; - int rval = 0, err; - gzFile gzfile; + fseek(file, file_offset, SEEK_SET); - /* level 1 is very close to 3 (the default) in terms of file size, - * but about twice as fast, best use for speedy saving - campbell */ - gzfile = BLI_gzopen(to, "wb1"); - if (gzfile == NULL) { - return -1; - } - file = BLI_open(from, O_BINARY | O_RDONLY, 0); - if (file == -1) { - return -2; - } + ZSTD_CCtx *ctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level); + + ZSTD_inBuffer input = {buf, len, 0}; - while (1) { - readsize = read(file, buffer, sizeof(buffer)); + size_t out_len = ZSTD_CStreamOutSize(); + void *out_buf = MEM_mallocN(out_len, __func__); + size_t total_written = 0; - if (readsize < 0) { - rval = -2; /* error happened in reading */ - fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno)); + /* Compress block and write it out until the input has been consumed. */ + while (input.pos < input.size) { + ZSTD_outBuffer output = {out_buf, out_len, 0}; + size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue); + if (ZSTD_isError(ret)) { break; } - else if (readsize == 0) { - break; /* done reading */ + if (fwrite(out_buf, 1, output.pos, file) != output.pos) { + break; } + total_written += output.pos; + } - if (gzwrite(gzfile, buffer, readsize) <= 0) { - rval = -1; /* error happened in writing */ - fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err)); + /* Finalize the Zstd frame. */ + size_t ret = 1; + while (ret != 0) { + ZSTD_outBuffer output = {out_buf, out_len, 0}; + ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end); + if (ZSTD_isError(ret)) { + break; + } + if (fwrite(out_buf, 1, output.pos, file) != output.pos) { break; } + total_written += output.pos; } - gzclose(gzfile); - close(file); + MEM_freeN(out_buf); + ZSTD_freeCCtx(ctx); - return rval; + return ZSTD_isError(ret) ? 0 : total_written; } -#endif -/* gzip the file in from_file and write it to memory to_mem, at most size bytes. - * return the unzipped size - */ -char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) +size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset) { - gzFile gzfile; - int readsize, size, alloc_size = 0; - char *mem = NULL; - const int chunk_size = 512 * 1024; + fseek(file, file_offset, SEEK_SET); - size = 0; + ZSTD_DCtx *ctx = ZSTD_createDCtx(); - gzfile = BLI_gzopen(from_file, "rb"); - for (;;) { - if (mem == NULL) { - mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem"); - alloc_size = chunk_size; - } - else { - mem = MEM_reallocN(mem, size + chunk_size); - alloc_size += chunk_size; - } + size_t in_len = ZSTD_DStreamInSize(); + void *in_buf = MEM_mallocN(in_len, __func__); + ZSTD_inBuffer input = {in_buf, in_len, 0}; - readsize = gzread(gzfile, mem + size, chunk_size); - if (readsize > 0) { - size += readsize; - } - else { + ZSTD_outBuffer output = {buf, len, 0}; + + size_t ret = 0; + /* Read and decompress chunks of input data until we have enough output. */ + while (output.pos < output.size && !ZSTD_isError(ret)) { + input.size = fread(in_buf, 1, in_len, file); + if (input.size == 0) { break; } - } - gzclose(gzfile); + /* Consume input data until we run out or have enough output. */ + input.pos = 0; + while (input.pos < input.size && output.pos < output.size) { + ret = ZSTD_decompressStream(ctx, &output, &input); - if (size == 0) { - MEM_freeN(mem); - mem = NULL; - } - else if (alloc_size != size) { - mem = MEM_reallocN(mem, size); + if (ZSTD_isError(ret)) { + break; + } + } } - *r_size = size; + MEM_freeN(in_buf); + ZSTD_freeDCtx(ctx); - return mem; + return ZSTD_isError(ret) ? 0 : output.pos; } -#define CHUNK (256 * 1024) - -/* gzip byte array from memory and write it to file at certain position. - * return size of gzip stream. - */ -size_t BLI_gzip_mem_to_file_at_pos( - void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level) +bool BLI_file_magic_is_gzip(const char header[4]) { - int ret, flush; - unsigned have; - z_stream strm; - unsigned char out[CHUNK]; - - BLI_fseek(file, gz_stream_offset, 0); - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - ret = deflateInit(&strm, compression_level); - if (ret != Z_OK) { - return 0; - } - - strm.avail_in = len; - strm.next_in = (Bytef *)buf; - flush = Z_FINISH; - - do { - strm.avail_out = CHUNK; - strm.next_out = out; - ret = deflate(&strm, flush); - if (ret == Z_STREAM_ERROR) { - return 0; - } - have = CHUNK - strm.avail_out; - if (fwrite(out, 1, have, file) != have || ferror(file)) { - deflateEnd(&strm); - return 0; - } - } while (strm.avail_out == 0); - - if (strm.avail_in != 0 || ret != Z_STREAM_END) { - return 0; - } - - deflateEnd(&strm); - return (size_t)strm.total_out; + /* GZIP itself starts with the magic bytes 0x1f 0x8b. + * The third byte indicates the compression method, which is 0x08 for DEFLATE. */ + return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08; } -/* read and decompress gzip stream from file at certain position to buffer. - * return size of decompressed data. - */ -size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) +bool BLI_file_magic_is_zstd(const char header[4]) { - int ret; - z_stream strm; - size_t chunk = 256 * 1024; - unsigned char in[CHUNK]; - - BLI_fseek(file, gz_stream_offset, 0); - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - ret = inflateInit(&strm); - if (ret != Z_OK) { - return 0; + /* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame. + * Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5* + * for skippable frames, with the * being anything from 0 to F. + * + * To check whether a file is Zstd-compressed, we just check whether the first frame matches + * either. Seeking through the file until a Zstd frame is found would make things more + * complicated and the probability of a false positive is rather low anyways. + * + * Note that LZ4 uses a compatible format, so even though its compressed frames have a + * different magic number, a valid LZ4 file might also start with a skippable frame matching + * the second check here. + * + * For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md + */ + + uint32_t magic = *((uint32_t *)header); + if (magic == 0xFD2FB528) { + return true; } - - do { - strm.avail_in = fread(in, 1, chunk, file); - strm.next_in = in; - if (ferror(file)) { - inflateEnd(&strm); - return 0; - } - - do { - strm.avail_out = len; - strm.next_out = (Bytef *)buf + strm.total_out; - - ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_ERROR) { - return 0; - } - } while (strm.avail_out == 0); - - } while (ret != Z_STREAM_END); - - inflateEnd(&strm); - return (size_t)strm.total_out; + if ((magic >> 4) == 0x184D2A5) { + return true; + } + return false; } -#undef CHUNK - /** * Returns true if the file with the specified name can be written. * This implementation uses access(2), which makes the check according diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c new file mode 100644 index 00000000000..3a833871e27 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_file.c @@ -0,0 +1,80 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#ifndef WIN32 +# include <unistd.h> /* for read close */ +#else +# include "BLI_winstuff.h" +# include "winsock2.h" +# include <io.h> /* for open close read */ +#endif + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + int filedes; +} RawFileReader; + +static ssize_t file_read(FileReader *reader, void *buffer, size_t size) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + ssize_t readsize = read(rawfile->filedes, buffer, size); + + if (readsize >= 0) { + rawfile->reader.offset += readsize; + } + + return readsize; +} + +static off64_t file_seek(FileReader *reader, off64_t offset, int whence) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence); + return rawfile->reader.offset; +} + +static void file_close(FileReader *reader) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + close(rawfile->filedes); + MEM_freeN(rawfile); +} + +FileReader *BLI_filereader_new_file(int filedes) +{ + RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__); + + rawfile->filedes = filedes; + + rawfile->reader.read = file_read; + rawfile->reader.seek = file_seek; + rawfile->reader.close = file_close; + + return (FileReader *)rawfile; +} diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c new file mode 100644 index 00000000000..72eb153a8b9 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_gzip.c @@ -0,0 +1,108 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include <zlib.h> + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + FileReader *base; + + z_stream strm; + + void *in_buf; + size_t in_size; +} GzipReader; + +static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size) +{ + GzipReader *gzip = (GzipReader *)reader; + + gzip->strm.avail_out = size; + gzip->strm.next_out = buffer; + + while (gzip->strm.avail_out > 0) { + if (gzip->strm.avail_in == 0) { + /* Ran out of buffered input data, read some more. */ + size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size); + + if (readsize > 0) { + /* We got some data, so mark the buffer as refilled. */ + gzip->strm.avail_in = readsize; + gzip->strm.next_in = gzip->in_buf; + } + else { + /* The underlying file is EOF, so return as much as we can. */ + break; + } + } + + int ret = inflate(&gzip->strm, Z_NO_FLUSH); + + if (ret != Z_OK && ret != Z_BUF_ERROR) { + break; + } + } + + ssize_t read_len = size - gzip->strm.avail_out; + gzip->reader.offset += read_len; + return read_len; +} + +static void gzip_close(FileReader *reader) +{ + GzipReader *gzip = (GzipReader *)reader; + + if (inflateEnd(&gzip->strm) != Z_OK) { + printf("close gzip stream error\n"); + } + MEM_freeN((void *)gzip->in_buf); + + gzip->base->close(gzip->base); + MEM_freeN(gzip); +} + +FileReader *BLI_filereader_new_gzip(FileReader *base) +{ + GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__); + gzip->base = base; + + if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) { + MEM_freeN(gzip); + return NULL; + } + + gzip->in_size = 256 * 2014; + gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf"); + + gzip->reader.read = gzip_read; + gzip->reader.seek = NULL; + gzip->reader.close = gzip_close; + + return (FileReader *)gzip; +} diff --git a/source/blender/blenlib/intern/filereader_memory.c b/source/blender/blenlib/intern/filereader_memory.c new file mode 100644 index 00000000000..150fed7d1cc --- /dev/null +++ b/source/blender/blenlib/intern/filereader_memory.c @@ -0,0 +1,145 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include <string.h> + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" +#include "BLI_mmap.h" + +#include "MEM_guardedalloc.h" + +/* This file implements both memory-backed and memory-mapped-file-backed reading. */ +typedef struct { + FileReader reader; + + const char *data; + BLI_mmap_file *mmap; + size_t length; +} MemoryReader; + +static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + memcpy(buffer, mem->data + mem->reader.offset, readsize); + mem->reader.offset += readsize; + + return readsize; +} + +static off64_t memory_seek(FileReader *reader, off64_t offset, int whence) +{ + MemoryReader *mem = (MemoryReader *)reader; + + off64_t new_pos; + if (whence == SEEK_CUR) { + new_pos = mem->reader.offset + offset; + } + else if (whence == SEEK_SET) { + new_pos = offset; + } + else if (whence == SEEK_END) { + new_pos = mem->length + offset; + } + else { + return -1; + } + + if (new_pos < 0 || new_pos > mem->length) { + return -1; + } + + mem->reader.offset = new_pos; + return mem->reader.offset; +} + +static void memory_close_raw(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLI_filereader_new_memory(const void *data, size_t len) +{ + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->data = (const char *)data; + mem->length = len; + + mem->reader.read = memory_read_raw; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_raw; + + return (FileReader *)mem; +} + +/* Memory-mapped file reading. + * By using `mmap()`, we can map a file so that it can be treated like normal memory, + * meaning that we can just read from it with `memcpy()` etc. + * This avoids system call overhead and can significantly speed up file loading. + */ + +static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) { + return 0; + } + + mem->reader.offset += readsize; + + return readsize; +} + +static void memory_close_mmap(FileReader *reader) +{ + MemoryReader *mem = (MemoryReader *)reader; + BLI_mmap_free(mem->mmap); + MEM_freeN(mem); +} + +FileReader *BLI_filereader_new_mmap(int filedes) +{ + BLI_mmap_file *mmap = BLI_mmap_open(filedes); + if (mmap == NULL) { + return NULL; + } + + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->mmap = mmap; + mem->length = BLI_lseek(filedes, 0, SEEK_END); + + mem->reader.read = memory_read_mmap; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_mmap; + + return (FileReader *)mem; +} diff --git a/source/blender/blenlib/intern/filereader_zstd.c b/source/blender/blenlib/intern/filereader_zstd.c new file mode 100644 index 00000000000..785a40cd1a1 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_zstd.c @@ -0,0 +1,335 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include <string.h> +#include <zstd.h> + +#include "BLI_blenlib.h" +#include "BLI_endian_switch.h" +#include "BLI_filereader.h" +#include "BLI_math_base.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + FileReader *base; + + ZSTD_DCtx *ctx; + ZSTD_inBuffer in_buf; + size_t in_buf_max_size; + + struct { + int num_frames; + size_t *compressed_ofs; + size_t *uncompressed_ofs; + + char *cached_content; + int cached_frame; + } seek; +} ZstdReader; + +static bool zstd_read_u32(FileReader *base, uint32_t *val) +{ + if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) { + return false; + } +#ifdef __BIG_ENDIAN__ + BLI_endian_switch_uint32(val); +#endif + return true; +} + +static bool zstd_read_seek_table(ZstdReader *zstd) +{ + FileReader *base = zstd->base; + + /* The seek table frame is at the end of the file, so seek there + * and verify that there is enough data. */ + if (base->seek(base, -4, SEEK_END) < 13) { + return false; + } + uint32_t magic; + if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) { + return false; + } + + uint8_t flags; + if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) { + return false; + } + /* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */ + bool has_checksums = (flags & 0x80); + if (flags & 0x60) { + return false; + } + + uint32_t num_frames; + if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &num_frames)) { + return false; + } + + /* Each frame has either 2 or 3 uint32_t, and after that we have + * num_frames, flags and magic for another 9 bytes. */ + uint32_t expected_frame_length = num_frames * (has_checksums ? 12 : 8) + 9; + /* The frame starts with another magic number and its length, but these + * two fields are not included when counting length. */ + off64_t frame_start_ofs = 8 + expected_frame_length; + /* Sanity check: Before the start of the seek table frame, + * there must be num_frames frames, each of which at least 8 bytes long. */ + off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END); + if (seek_frame_start < num_frames * 8) { + return false; + } + + if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) { + return false; + } + + uint32_t frame_length; + if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) { + return false; + } + + zstd->seek.num_frames = num_frames; + zstd->seek.compressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__); + zstd->seek.uncompressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__); + + size_t compressed_ofs = 0; + size_t uncompressed_ofs = 0; + for (int i = 0; i < num_frames; i++) { + uint32_t compressed_size, uncompressed_size; + if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) { + break; + } + if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) { + break; + } + zstd->seek.compressed_ofs[i] = compressed_ofs; + zstd->seek.uncompressed_ofs[i] = uncompressed_ofs; + compressed_ofs += compressed_size; + uncompressed_ofs += uncompressed_size; + } + zstd->seek.compressed_ofs[num_frames] = compressed_ofs; + zstd->seek.uncompressed_ofs[num_frames] = uncompressed_ofs; + + /* Seek to the end of the previous frame for the following BHead frame detection. */ + if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) { + MEM_freeN(zstd->seek.compressed_ofs); + MEM_freeN(zstd->seek.uncompressed_ofs); + memset(&zstd->seek, 0, sizeof(zstd->seek)); + return false; + } + + zstd->seek.cached_frame = -1; + + return true; +} + +/* Find out which frame contains the given position in the uncompressed stream. + * Basically just bisection. */ +static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos) +{ + int low = 0, high = zstd->seek.num_frames; + + if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) { + return -1; + } + + while (low + 1 < high) { + int mid = low + ((high - low) >> 1); + if (zstd->seek.uncompressed_ofs[mid] <= pos) { + low = mid; + } + else { + high = mid; + } + } + + return low; +} + +/* Ensure that the currently loaded frame is the correct one. */ +static const char *zstd_ensure_cache(ZstdReader *zstd, int frame) +{ + if (zstd->seek.cached_frame == frame) { + /* Cached frame matches, so just return it. */ + return zstd->seek.cached_content; + } + + /* Cached frame doesn't match, so discard it and cache the wanted one onstead. */ + MEM_SAFE_FREE(zstd->seek.cached_content); + + size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame]; + size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] - + zstd->seek.uncompressed_ofs[frame]; + + char *uncompressed_data = MEM_mallocN(uncompressed_size, __func__); + char *compressed_data = MEM_mallocN(compressed_size, __func__); + if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 || + zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size) { + MEM_freeN(compressed_data); + MEM_freeN(uncompressed_data); + return NULL; + } + + size_t res = ZSTD_decompressDCtx( + zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size); + MEM_freeN(compressed_data); + if (ZSTD_isError(res) || res < uncompressed_size) { + MEM_freeN(uncompressed_data); + return NULL; + } + + zstd->seek.cached_frame = frame; + zstd->seek.cached_content = uncompressed_data; + return uncompressed_data; +} + +static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size) +{ + ZstdReader *zstd = (ZstdReader *)reader; + + size_t end_offset = zstd->reader.offset + size, read_len = 0; + while (zstd->reader.offset < end_offset) { + int frame = zstd_frame_from_pos(zstd, zstd->reader.offset); + if (frame < 0) { + /* EOF is reached, so return as much as we can. */ + break; + } + + const char *framedata = zstd_ensure_cache(zstd, frame); + if (framedata == NULL) { + /* Error while reading the frame, so return as much as we can. */ + break; + } + + size_t frame_end_offset = min_zz(zstd->seek.uncompressed_ofs[frame + 1], end_offset); + size_t frame_read_len = frame_end_offset - zstd->reader.offset; + + size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame]; + memcpy((char *)buffer + read_len, framedata + offset_in_frame, frame_read_len); + read_len += frame_read_len; + zstd->reader.offset = frame_end_offset; + } + + return read_len; +} + +static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence) +{ + ZstdReader *zstd = (ZstdReader *)reader; + off64_t new_pos; + if (whence == SEEK_SET) { + new_pos = offset; + } + else if (whence == SEEK_END) { + new_pos = zstd->seek.uncompressed_ofs[zstd->seek.num_frames] + offset; + } + else { + new_pos = zstd->reader.offset + offset; + } + + if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) { + return -1; + } + zstd->reader.offset = new_pos; + return zstd->reader.offset; +} + +static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size) +{ + ZstdReader *zstd = (ZstdReader *)reader; + ZSTD_outBuffer output = {buffer, size, 0}; + + while (output.pos < output.size) { + if (zstd->in_buf.pos == zstd->in_buf.size) { + /* Ran out of buffered input data, read some more. */ + zstd->in_buf.pos = 0; + ssize_t readsize = zstd->base->read( + zstd->base, (char *)zstd->in_buf.src, zstd->in_buf_max_size); + + if (readsize > 0) { + /* We got some data, so mark the buffer as refilled. */ + zstd->in_buf.size = readsize; + } + else { + /* The underlying file is EOF, so return as much as we can. */ + break; + } + } + + if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) { + break; + } + } + + zstd->reader.offset += output.pos; + return output.pos; +} + +static void zstd_close(FileReader *reader) +{ + ZstdReader *zstd = (ZstdReader *)reader; + + ZSTD_freeDCtx(zstd->ctx); + if (zstd->reader.seek) { + MEM_freeN(zstd->seek.uncompressed_ofs); + MEM_freeN(zstd->seek.compressed_ofs); + MEM_freeN(zstd->seek.cached_content); + } + else { + MEM_freeN((void *)zstd->in_buf.src); + } + + zstd->base->close(zstd->base); + MEM_freeN(zstd); +} + +FileReader *BLI_filereader_new_zstd(FileReader *base) +{ + ZstdReader *zstd = MEM_callocN(sizeof(ZstdReader), __func__); + + zstd->ctx = ZSTD_createDCtx(); + zstd->base = base; + + if (zstd_read_seek_table(zstd)) { + zstd->reader.read = zstd_read_seekable; + zstd->reader.seek = zstd_seek; + } + else { + zstd->reader.read = zstd_read; + zstd->reader.seek = NULL; + + zstd->in_buf_max_size = ZSTD_DStreamInSize(); + zstd->in_buf.src = MEM_mallocN(zstd->in_buf_max_size, "zstd in buf"); + zstd->in_buf.size = zstd->in_buf_max_size; + /* This signals that the buffer has run out, + * which will make the read function refill it on the first call. */ + zstd->in_buf.pos = zstd->in_buf_max_size; + } + zstd->reader.close = zstd_close; + + return (FileReader *)zstd; +} diff --git a/source/blender/blenloader/BLO_undofile.h b/source/blender/blenloader/BLO_undofile.h index fc41a6e832f..4e240e2462b 100644 --- a/source/blender/blenloader/BLO_undofile.h +++ b/source/blender/blenloader/BLO_undofile.h @@ -24,6 +24,8 @@ * \ingroup blenloader */ +#include "BLI_filereader.h" + struct GHash; struct Scene; @@ -65,6 +67,16 @@ typedef struct MemFileUndoData { size_t undo_size; } MemFileUndoData; +/* FileReader-compatible wrapper for reading MemFiles */ +typedef struct { + FileReader reader; + + MemFile *memfile; + int undo_direction; + + bool memchunk_identical; +} UndoReader; + /* actually only used writefile.c */ void BLO_memfile_write_init(MemFileWriteData *mem_data, @@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile, struct Main *bmain, struct Scene **r_scene); extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename); + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction); diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt index f5baf0dcb83..89631588ed0 100644 --- a/source/blender/blenloader/CMakeLists.txt +++ b/source/blender/blenloader/CMakeLists.txt @@ -42,7 +42,7 @@ set(INC ) set(INC_SYS - ${ZLIB_INCLUDE_DIRS} + ${ZSTD_INCLUDE_DIRS} ) set(SRC diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index e48c305fc4b..49c3497f996 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -21,8 +21,6 @@ * \ingroup blenloader */ -#include "zlib.h" - #include <ctype.h> /* for isdigit. */ #include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */ #include <limits.h> @@ -71,7 +69,6 @@ #include "BLI_math.h" #include "BLI_memarena.h" #include "BLI_mempool.h" -#include "BLI_mmap.h" #include "BLI_threads.h" #include "PIL_time.h" @@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd) */ if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) { bhead4.code = DATA; - readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL); + readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4)); if (readsize == sizeof(bhead4) || bhead4.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd) } else { bhead8.code = DATA; - readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL); + readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8)); if (readsize == sizeof(bhead8) || bhead8.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd) /* pass */ } #ifdef USE_BHEAD_READ_ON_DEMAND - else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { + else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { /* Delay reading bhead content. */ new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead"); if (new_bhead) { new_bhead->next = new_bhead->prev = NULL; - new_bhead->file_offset = fd->file_offset; + new_bhead->file_offset = fd->file->offset; new_bhead->has_data = false; new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR); + off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR); if (seek_new == -1) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } - BLI_assert(fd->file_offset == seek_new); + BLI_assert(fd->file->offset == seek_new); } else { fd->is_eof = true; @@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd) new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - readsize = fd->read( - fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical); + readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len); - if (readsize != (ssize_t)bhead.len) { + if (readsize != bhead.len) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } + + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } else { fd->is_eof = true; @@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf) bool success = true; BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock); BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0); - off64_t offset_backup = fd->file_offset; - if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) { + off64_t offset_backup = fd->file->offset; + if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) { success = false; } else { - if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) != - (ssize_t)new_bhead->bhead.len) { + if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) { success = false; } + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } - if (fd->seek(fd, offset_backup, SEEK_SET) == -1) { + if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) { success = false; } return success; @@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd) ssize_t readsize; /* read in the header data */ - readsize = fd->read(fd, header, sizeof(header), NULL); + readsize = fd->file->read(fd->file, header, sizeof(header)); if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') && ELEM(header[8], 'v', 'V') && @@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd) /** \} */ -/* -------------------------------------------------------------------- */ -/** \name File Data API - * \{ */ - -/* Regular file reading. */ - -static ssize_t fd_read_data_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - ssize_t readsize = read(filedata->filedes, buffer, size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence) -{ - filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence); - return filedata->file_offset; -} - -/* GZip file reading. */ - -static ssize_t fd_read_gzip_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - BLI_assert(size <= INT_MAX); - - ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -/* Memory reading. */ - -static ssize_t fd_read_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset); - - memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize); - filedata->file_offset += readsize; - - return readsize; -} - -/* Memory-mapped file reading. - * By using mmap(), we can map a file so that it can be treated like normal memory, - * meaning that we can just read from it with memcpy() etc. - * This avoids system call overhead and can significantly speed up file loading. - */ - -static ssize_t fd_read_from_mmap(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset)); - - if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) { - return 0; - } - - filedata->file_offset += readsize; - - return readsize; -} - -static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence) -{ - off64_t new_pos; - if (whence == SEEK_CUR) { - new_pos = filedata->file_offset + offset; - } - else if (whence == SEEK_SET) { - new_pos = offset; - } - else if (whence == SEEK_END) { - new_pos = filedata->buffersize + offset; - } - else { - return -1; - } - - if (new_pos < 0 || new_pos > filedata->buffersize) { - return -1; - } - - filedata->file_offset = new_pos; - return filedata->file_offset; -} - -/* MemFile reading. */ - -static ssize_t fd_read_from_memfile(FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunck_identical) -{ - static size_t seek = SIZE_MAX; /* the current position */ - static size_t offset = 0; /* size of previous chunks */ - static MemFileChunk *chunk = NULL; - size_t chunkoffset, readsize, totread; - - if (r_is_memchunck_identical != NULL) { - *r_is_memchunck_identical = true; - } - - if (size == 0) { - return 0; - } - - if (seek != (size_t)filedata->file_offset) { - chunk = filedata->memfile->chunks.first; - seek = 0; - - while (chunk) { - if (seek + chunk->size > (size_t)filedata->file_offset) { - break; - } - seek += chunk->size; - chunk = chunk->next; - } - offset = seek; - seek = (size_t)filedata->file_offset; - } - - if (chunk) { - totread = 0; - - do { - /* first check if it's on the end if current chunk */ - if (seek - offset == chunk->size) { - offset += chunk->size; - chunk = chunk->next; - } - - /* debug, should never happen */ - if (chunk == NULL) { - CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk"); - return 0; - } - - chunkoffset = seek - offset; - readsize = size - totread; - - /* data can be spread over multiple chunks, so clamp size - * to within this chunk, and then it will read further in - * the next chunk */ - if (chunkoffset + readsize > chunk->size) { - readsize = chunk->size - chunkoffset; - } - - memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); - totread += readsize; - filedata->file_offset += readsize; - seek += readsize; - if (r_is_memchunck_identical != NULL) { - /* `is_identical` of current chunk represents whether it changed compared to previous undo - * step. this is fine in redo case, but not in undo case, where we need an extra flag - * defined when saving the next (future) step after the one we want to restore, as we are - * supposed to 'come from' that future undo step, and not the one before current one. */ - *r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ? - chunk->is_identical : - chunk->is_identical_future; - } - } while (totread < size); - - return (ssize_t)totread; - } - - return 0; -} - static FileData *filedata_new(BlendFileReadReport *reports) { BLI_assert(reports != NULL); FileData *fd = MEM_callocN(sizeof(FileData), "FileData"); - fd->filedes = -1; - fd->gzfiledes = NULL; - fd->memsdna = DNA_sdna_current_get(); fd->datamap = oldnewmap_new(); @@ -1387,78 +1191,66 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports) static FileData *blo_filedata_from_file_descriptor(const char *filepath, BlendFileReadReport *reports, - int file) + int filedes) { - FileDataReadFn *read_fn = NULL; - FileDataSeekFn *seek_fn = NULL; /* Optional. */ - size_t buffersize = 0; - BLI_mmap_file *mmap_file = NULL; - - gzFile gzfile = (gzFile)Z_NULL; - char header[7]; + FileReader *rawfile = BLI_filereader_new_file(filedes); + FileReader *file = NULL; - /* Regular file. */ errno = 0; - if (read(file, header, sizeof(header)) != sizeof(header)) { + /* If opening the file failed or we can't read the header, give up. */ + if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) { BKE_reportf(reports->reports, RPT_WARNING, "Unable to read '%s': %s", filepath, errno ? strerror(errno) : TIP_("insufficient content")); + if (rawfile) { + rawfile->close(rawfile); + } + else { + close(filedes); + } return NULL; } - /* Regular file. */ - if (memcmp(header, "BLENDER", sizeof(header)) == 0) { - read_fn = fd_read_data_from_file; - seek_fn = fd_seek_data_from_file; + /* Rewind the file after reading the header. */ + rawfile->seek(rawfile, 0, SEEK_SET); - mmap_file = BLI_mmap_open(file); - if (mmap_file != NULL) { - read_fn = fd_read_from_mmap; - seek_fn = fd_seek_from_mmap; - buffersize = BLI_lseek(file, 0, SEEK_END); + /* Check if we have a regular file. */ + if (memcmp(header, "BLENDER", sizeof(header)) == 0) { + /* Try opening the file with memory-mapped IO. */ + file = BLI_filereader_new_mmap(filedes); + if (file == NULL) { + /* mmap failed, so just keep using rawfile. */ + file = rawfile; + rawfile = NULL; } } - - BLI_lseek(file, 0, SEEK_SET); - - /* Gzip file. */ - errno = 0; - if ((read_fn == NULL) && - /* Check header magic. */ - (header[0] == 0x1f && header[1] == 0x8b)) { - gzfile = BLI_gzopen(filepath, "rb"); - if (gzfile == (gzFile)Z_NULL) { - BKE_reportf(reports->reports, - RPT_WARNING, - "Unable to open '%s': %s", - filepath, - errno ? strerror(errno) : TIP_("unknown error reading file")); - return NULL; + else if (BLI_file_magic_is_gzip(header)) { + file = BLI_filereader_new_gzip(rawfile); + if (file != NULL) { + rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */ + } + } + else if (BLI_file_magic_is_zstd(header)) { + file = BLI_filereader_new_zstd(rawfile); + if (file != NULL) { + rawfile = NULL; /* The Zstd FileReader takes ownership of `rawfile`. */ } - - /* 'seek_fn' is too slow for gzip, don't set it. */ - read_fn = fd_read_gzip_from_file; - /* Caller must close. */ - file = -1; } - if (read_fn == NULL) { + /* Clean up `rawfile` if it wasn't taken over. */ + if (rawfile != NULL) { + rawfile->close(rawfile); + } + if (file == NULL) { BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath); return NULL; } FileData *fd = filedata_new(reports); - - fd->filedes = file; - fd->gzfiledes = gzfile; - - fd->read = read_fn; - fd->seek = seek_fn; - fd->mmap_file = mmap_file; - fd->buffersize = buffersize; + fd->file = file; return fd; } @@ -1475,11 +1267,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead errno ? strerror(errno) : TIP_("unknown error reading file")); return NULL; } - FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file); - if ((fd == NULL) || (fd->filedes == -1)) { - close(file); - } - return fd; + return blo_filedata_from_file_descriptor(filepath, reports, file); } /* cannot be called with relative paths anymore! */ @@ -1513,50 +1301,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath) return NULL; } -static ssize_t fd_read_gzip_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - int err; - - filedata->strm.next_out = (Bytef *)buffer; - filedata->strm.avail_out = (uint)size; - - /* Inflate another chunk. */ - err = inflate(&filedata->strm, Z_SYNC_FLUSH); - - if (err == Z_STREAM_END) { - return 0; - } - if (err != Z_OK) { - CLOG_ERROR(&LOG, "ZLib error (code %d)", err); - return 0; - } - - filedata->file_offset += size; - - return (ssize_t)size; -} - -static int fd_read_gzip_from_memory_init(FileData *fd) -{ - - fd->strm.next_in = (Bytef *)fd->buffer; - fd->strm.avail_in = fd->buffersize; - fd->strm.total_out = 0; - fd->strm.zalloc = Z_NULL; - fd->strm.zfree = Z_NULL; - - if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) { - return 0; - } - - fd->read = fd_read_gzip_from_memory; - - return 1; -} - FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports) { if (!mem || memsize < SIZEOFBLENDERHEADER) { @@ -1565,24 +1309,24 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe return NULL; } - FileData *fd = filedata_new(reports); - const char *cp = mem; - - fd->buffer = mem; - fd->buffersize = memsize; + FileReader *mem_file = BLI_filereader_new_memory(mem, memsize); + FileReader *file = mem_file; - /* test if gzip */ - if (cp[0] == 0x1f && cp[1] == 0x8b) { - if (0 == fd_read_gzip_from_memory_init(fd)) { - blo_filedata_free(fd); - return NULL; - } + if (BLI_file_magic_is_gzip(mem)) { + file = BLI_filereader_new_gzip(mem_file); } - else { - fd->read = fd_read_from_memory; + else if (BLI_file_magic_is_zstd(mem)) { + file = BLI_filereader_new_zstd(mem_file); } - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + if (file == NULL) { + /* Compression initialization failed. */ + mem_file->close(mem_file); + return NULL; + } + + FileData *fd = filedata_new(reports); + fd->file = file; return blo_decode_and_check(fd, reports->reports); } @@ -1597,11 +1341,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, } FileData *fd = filedata_new(reports); - fd->memfile = memfile; + fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction); fd->undo_direction = params->undo_direction; - - fd->read = fd_read_from_memfile; - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + fd->flags |= FD_FLAGS_IS_MEMFILE; return blo_decode_and_check(fd, reports->reports); } @@ -1609,30 +1351,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, void blo_filedata_free(FileData *fd) { if (fd) { - if (fd->filedes != -1) { - close(fd->filedes); - } - - if (fd->gzfiledes != NULL) { - gzclose(fd->gzfiledes); - } - - if (fd->strm.next_in) { - int err = inflateEnd(&fd->strm); - if (err != Z_OK) { - CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err); - } - } - - if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) { - MEM_freeN((void *)fd->buffer); - fd->buffer = NULL; - } - - if (fd->mmap_file) { - BLI_mmap_free(fd->mmap_file); - fd->mmap_file = NULL; - } + fd->file->close(fd->file); /* Free all BHeadN data blocks */ #ifndef NDEBUG @@ -1640,7 +1359,7 @@ void blo_filedata_free(FileData *fd) #else /* Sanity check we're not keeping memory we don't need. */ LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) { - if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { + if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { BLI_assert(new_bhead->has_data == 0); } MEM_freeN(new_bhead); @@ -2096,7 +1815,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id), void blo_cache_storage_init(FileData *fd, Main *bmain) { - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { BLI_assert(fd->cache_storage == NULL); fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__); fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__); @@ -2261,7 +1980,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname) * undo since DNA must match. */ static const void *peek_struct_undo(FileData *fd, BHead *bhead) { - BLI_assert(fd->memfile != NULL); + BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE); UNUSED_VARS_NDEBUG(fd); return (bhead->len) ? (const void *)(bhead + 1) : NULL; } @@ -3679,7 +3398,7 @@ static BHead *read_libblock(FileData *fd, * When datablocks are changed but still exist, we restore them at the old * address and inherit recalc flags for the dependency graph. */ ID *id_old = NULL; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) { if (r_id) { *r_id = id_old; @@ -3980,13 +3699,14 @@ static void lib_link_all(FileData *fd, Main *bmain) continue; } - if (fd->memfile != NULL && GS(id->name) == ID_WM) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) { /* No load UI for undo memfiles. * Only WM currently, SCR needs it still (see below), and so does WS? */ continue; } - if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo && + (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { /* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across * current undo step, and old IDs re-use their old memory address, we do not need to liblink * it at all. */ @@ -4165,7 +3885,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) BlendFileData *bfd; ListBase mainlist = {NULL, NULL}; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step"); } @@ -4256,7 +3976,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } /* do before read_libraries, but skip undo case */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) { do_versions(fd, NULL, bfd->main); } @@ -4278,7 +3998,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries; /* Skip in undo case. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Note that we can't recompute user-counts at this point in undo case, we play too much with * IDs from different memory realms, and Main database is not in a fully valid state yet. */ @@ -4311,7 +4031,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) /* Now that all our data-blocks are loaded, * we can re-generate overrides from their references. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Do not apply in undo case! */ fd->reports->duration.lib_overrides = PIL_check_seconds_timer(); @@ -4391,7 +4111,7 @@ static void sort_bhead_old_map(FileData *fd) static BHead *find_previous_lib(FileData *fd, BHead *bhead) { /* Skip library data-blocks in undo, see comment in read_libblock. */ - if (fd->memfile) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { return NULL; } @@ -5850,7 +5570,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p) bool BLO_read_data_is_undo(BlendDataReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr) @@ -5870,7 +5590,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader) bool BLO_read_lib_is_undo(BlendLibReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } Main *BLO_read_lib_get_main(BlendLibReader *reader) diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h index b04043f9641..beeed8e45ae 100644 --- a/source/blender/blenloader/intern/readfile.h +++ b/source/blender/blenloader/intern/readfile.h @@ -28,10 +28,10 @@ # include "BLI_winstuff.h" #endif +#include "BLI_filereader.h" #include "DNA_sdna_types.h" #include "DNA_space_types.h" #include "DNA_windowmanager_types.h" /* for ReportType */ -#include "zlib.h" struct BLI_mmap_file; struct BLOCacheStorage; @@ -50,7 +50,7 @@ enum eFileDataFlag { FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1, FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2, FD_FLAGS_FILE_OK = 1 << 3, - FD_FLAGS_NOT_MY_BUFFER = 1 << 4, + FD_FLAGS_IS_MEMFILE = 1 << 4, /* XXX Unused in practice (checked once but never set). */ FD_FLAGS_NOT_MY_LIBMAP = 1 << 5, }; @@ -60,44 +60,18 @@ enum eFileDataFlag { # pragma GCC poison off_t #endif -#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__) -typedef int64_t off64_t; -#endif - -typedef ssize_t(FileDataReadFn)(struct FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunk_identical); -typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence); - typedef struct FileData { /** Linked list of BHeadN's. */ ListBase bhead_list; enum eFileDataFlag flags; bool is_eof; - size_t buffersize; - off64_t file_offset; - FileDataReadFn *read; - FileDataSeekFn *seek; + FileReader *file; - /** Regular file reading. */ - int filedes; - - /** Variables needed for reading from memory / stream / memory-mapped files. */ - const char *buffer; - struct BLI_mmap_file *mmap_file; - /** Variables needed for reading from memfile (undo). */ - struct MemFile *memfile; /** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use * to detect unchanged data from memfile. */ int undo_direction; /* eUndoStepDir */ - /** Variables needed for reading from file. */ - gzFile gzfiledes; - /** Gzip stream for memory decompression. */ - z_stream strm; - /** Now only in use for library appending. */ char relabase[FILE_MAX]; diff --git a/source/blender/blenloader/intern/undofile.c b/source/blender/blenloader/intern/undofile.c index 2eeeac2e8d7..62072cf7df5 100644 --- a/source/blender/blenloader/intern/undofile.c +++ b/source/blender/blenloader/intern/undofile.c @@ -48,6 +48,7 @@ #include "BKE_lib_id.h" #include "BKE_main.h" +#include "BKE_undo_system.h" /* keep last */ #include "BLI_strict_flags.h" @@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename) } return true; } + +static ssize_t undo_read(FileReader *reader, void *buffer, size_t size) +{ + UndoReader *undo = (UndoReader *)reader; + + static size_t seek = SIZE_MAX; /* The current position. */ + static size_t offset = 0; /* Size of previous chunks. */ + static MemFileChunk *chunk = NULL; + size_t chunkoffset, readsize, totread; + + undo->memchunk_identical = true; + + if (size == 0) { + return 0; + } + + if (seek != (size_t)undo->reader.offset) { + chunk = undo->memfile->chunks.first; + seek = 0; + + while (chunk) { + if (seek + chunk->size > (size_t)undo->reader.offset) { + break; + } + seek += chunk->size; + chunk = chunk->next; + } + offset = seek; + seek = (size_t)undo->reader.offset; + } + + if (chunk) { + totread = 0; + + do { + /* First check if it's on the end if current chunk. */ + if (seek - offset == chunk->size) { + offset += chunk->size; + chunk = chunk->next; + } + + /* Debug, should never happen. */ + if (chunk == NULL) { + printf("illegal read, chunk zero\n"); + return 0; + } + + chunkoffset = seek - offset; + readsize = size - totread; + + /* Data can be spread over multiple chunks, so clamp size + * to within this chunk, and then it will read further in + * the next chunk. */ + if (chunkoffset + readsize > chunk->size) { + readsize = chunk->size - chunkoffset; + } + + memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); + totread += readsize; + undo->reader.offset += (off64_t)readsize; + seek += readsize; + + /* `is_identical` of current chunk represents whether it changed compared to previous undo + * step. this is fine in redo case, but not in undo case, where we need an extra flag + * defined when saving the next (future) step after the one we want to restore, as we are + * supposed to 'come from' that future undo step, and not the one before current one. */ + undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical : + chunk->is_identical_future; + } while (totread < size); + + return (ssize_t)totread; + } + + return 0; +} + +static void undo_close(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction) +{ + UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__); + + undo->memfile = memfile; + undo->undo_direction = undo_direction; + + undo->reader.read = undo_read; + undo->reader.seek = NULL; + undo->reader.close = undo_close; + + return (FileReader *)undo; +} diff --git a/source/blender/blenloader/intern/versioning_250.c b/source/blender/blenloader/intern/versioning_250.c index e56c1995363..436645c2241 100644 --- a/source/blender/blenloader/intern/versioning_250.c +++ b/source/blender/blenloader/intern/versioning_250.c @@ -23,8 +23,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include <io.h> /* for open close read */ -# include <zlib.h> /* odd include order-issue */ +# include <io.h> /* for open close read */ #endif /* allow readfile to use deprecated functionality */ diff --git a/source/blender/blenloader/intern/versioning_legacy.c b/source/blender/blenloader/intern/versioning_legacy.c index 81371e1c1ed..6ba27b6ee9e 100644 --- a/source/blender/blenloader/intern/versioning_legacy.c +++ b/source/blender/blenloader/intern/versioning_legacy.c @@ -28,8 +28,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include <io.h> /* for open close read */ -# include <zlib.h> /* odd include order-issue */ +# include <io.h> /* for open close read */ #endif /* allow readfile to use deprecated functionality */ diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c index 99246603e9a..90d58514eb5 100644 --- a/source/blender/blenloader/intern/writefile.c +++ b/source/blender/blenloader/intern/writefile.c @@ -83,7 +83,6 @@ # include "BLI_winstuff.h" # include "winsock2.h" # include <io.h> -# include <zlib.h> /* odd include order-issue */ #else # include <unistd.h> /* FreeBSD, for write() and close(). */ #endif @@ -101,7 +100,12 @@ #include "BLI_bitmap.h" #include "BLI_blenlib.h" #include "BLI_endian_defines.h" +#include "BLI_endian_switch.h" +#include "BLI_link_utils.h" +#include "BLI_linklist.h" +#include "BLI_math_base.h" #include "BLI_mempool.h" +#include "BLI_threads.h" #include "MEM_guardedalloc.h" /* MEM_freeN */ #include "BKE_blender_version.h" @@ -129,14 +133,21 @@ #include <errno.h> +#include <zstd.h> + /* Make preferences read-only. */ #define U (*((const UserDef *)&U)) /* ********* my write, buffered writing with minimum size chunks ************ */ /* Use optimal allocation since blocks of this size are kept in memory for undo. */ -#define MYWRITE_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */ -#define MYWRITE_MAX_CHUNK (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */ +#define MEM_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */ +#define MEM_CHUNK_SIZE (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */ + +#define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */ +#define ZSTD_CHUNK_SIZE (1 << 20) /* 1mb */ + +#define ZSTD_COMPRESSION_LEVEL 3 /** Use if we want to store how many bytes have been written to the file. */ // #define USE_WRITE_DATA_LEN @@ -147,9 +158,16 @@ typedef enum { WW_WRAP_NONE = 1, - WW_WRAP_ZLIB, + WW_WRAP_ZSTD, } eWriteWrapType; +typedef struct ZstdFrame { + struct ZstdFrame *next, *prev; + + uint32_t compressed_size; + uint32_t uncompressed_size; +} ZstdFrame; + typedef struct WriteWrap WriteWrap; struct WriteWrap { /* callbacks */ @@ -161,15 +179,23 @@ struct WriteWrap { bool use_buf; /* internal */ - union { - int file_handle; - gzFile gz_handle; - } _user_data; + int file_handle; + struct { + ListBase threadpool; + ListBase tasks; + ThreadMutex mutex; + ThreadCondition condition; + int next_frame; + int num_frames; + + int level; + ListBase frames; + + bool write_error; + } zstd; }; /* none */ -#define FILE_HANDLE(ww) (ww)->_user_data.file_handle - static bool ww_open_none(WriteWrap *ww, const char *filepath) { int file; @@ -177,7 +203,7 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath) file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666); if (file != -1) { - FILE_HANDLE(ww) = file; + ww->file_handle = file; return true; } @@ -185,39 +211,170 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath) } static bool ww_close_none(WriteWrap *ww) { - return (close(FILE_HANDLE(ww)) != -1); + return (close(ww->file_handle) != -1); } static size_t ww_write_none(WriteWrap *ww, const char *buf, size_t buf_len) { - return write(FILE_HANDLE(ww), buf, buf_len); + return write(ww->file_handle, buf, buf_len); } -#undef FILE_HANDLE -/* zlib */ -#define FILE_HANDLE(ww) (ww)->_user_data.gz_handle +/* zstd */ + +typedef struct { + struct ZstdWriteBlockTask *next, *prev; + void *data; + size_t size; + int frame_number; + WriteWrap *ww; +} ZstdWriteBlockTask; -static bool ww_open_zlib(WriteWrap *ww, const char *filepath) +static void *zstd_write_task(void *userdata) { - gzFile file; + ZstdWriteBlockTask *task = userdata; + WriteWrap *ww = task->ww; - file = BLI_gzopen(filepath, "wb1"); + size_t out_buf_len = ZSTD_compressBound(task->size); + void *out_buf = MEM_mallocN(out_buf_len, "Zstd out buffer"); + size_t out_size = ZSTD_compress( + out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL); - if (file != Z_NULL) { - FILE_HANDLE(ww) = file; - return true; + MEM_freeN(task->data); + + BLI_mutex_lock(&ww->zstd.mutex); + + while (ww->zstd.next_frame != task->frame_number) { + BLI_condition_wait(&ww->zstd.condition, &ww->zstd.mutex); } - return false; + if (ZSTD_isError(out_size)) { + ww->zstd.write_error = true; + } + else { + if (ww_write_none(ww, out_buf, out_size) == out_size) { + ZstdFrame *frameinfo = MEM_mallocN(sizeof(ZstdFrame), "zstd frameinfo"); + frameinfo->uncompressed_size = task->size; + frameinfo->compressed_size = out_size; + BLI_addtail(&ww->zstd.frames, frameinfo); + } + else { + ww->zstd.write_error = true; + } + } + + ww->zstd.next_frame++; + + BLI_mutex_unlock(&ww->zstd.mutex); + BLI_condition_notify_all(&ww->zstd.condition); + + MEM_freeN(out_buf); + return NULL; +} + +static bool ww_open_zstd(WriteWrap *ww, const char *filepath) +{ + if (!ww_open_none(ww, filepath)) { + return false; + } + + /* Leave one thread open for the main writing logic, unless we only have one HW thread. */ + int num_threads = max_ii(1, BLI_system_thread_count() - 1); + BLI_threadpool_init(&ww->zstd.threadpool, zstd_write_task, num_threads); + BLI_mutex_init(&ww->zstd.mutex); + BLI_condition_init(&ww->zstd.condition); + + return true; } -static bool ww_close_zlib(WriteWrap *ww) + +static void zstd_write_u32_le(WriteWrap *ww, uint32_t val) +{ +#ifdef __BIG_ENDIAN__ + BLI_endian_switch_uint32(&val); +#endif + ww_write_none(ww, (char *)&val, sizeof(uint32_t)); +} + +/* In order to implement efficient seeking when reading the .blend, we append + * a skippable frame that encodes information about the other frames present + * in the file. + * The format here follows the upstream spec for seekable files: + * https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md + * If this information is not present in a file (e.g. if it was compressed + * with external tools), it can still be opened in Blender, but seeking will + * not be supported, so more memory might be needed. */ +static void zstd_write_seekable_frames(WriteWrap *ww) +{ + /* Write seek table header (magic number and frame size). */ + zstd_write_u32_le(ww, 0x184D2A5E); + + /* The actual frame number might not match ww->zstd.num_frames if there was a write error. */ + const uint32_t num_frames = BLI_listbase_count(&ww->zstd.frames); + /* Each frame consists of two u32, so 8 bytes each. + * After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */ + const uint32_t frame_size = num_frames * 8 + 9; + zstd_write_u32_le(ww, frame_size); + + /* Write seek table entries. */ + LISTBASE_FOREACH (ZstdFrame *, frame, &ww->zstd.frames) { + zstd_write_u32_le(ww, frame->compressed_size); + zstd_write_u32_le(ww, frame->uncompressed_size); + } + + /* Write seek table footer (number of frames, option flags and second magic number). */ + zstd_write_u32_le(ww, num_frames); + const char flags = 0; /* We don't store checksums for each frame. */ + ww_write_none(ww, &flags, 1); + zstd_write_u32_le(ww, 0x8F92EAB1); +} + +static bool ww_close_zstd(WriteWrap *ww) { - return (gzclose(FILE_HANDLE(ww)) == Z_OK); + BLI_threadpool_end(&ww->zstd.threadpool); + BLI_freelistN(&ww->zstd.tasks); + + BLI_mutex_end(&ww->zstd.mutex); + BLI_condition_end(&ww->zstd.condition); + + zstd_write_seekable_frames(ww); + BLI_freelistN(&ww->zstd.frames); + + return ww_close_none(ww) && !ww->zstd.write_error; } -static size_t ww_write_zlib(WriteWrap *ww, const char *buf, size_t buf_len) + +static size_t ww_write_zstd(WriteWrap *ww, const char *buf, size_t buf_len) { - return gzwrite(FILE_HANDLE(ww), buf, buf_len); + if (ww->zstd.write_error) { + return 0; + } + + ZstdWriteBlockTask *task = MEM_mallocN(sizeof(ZstdWriteBlockTask), __func__); + task->data = MEM_mallocN(buf_len, __func__); + memcpy(task->data, buf, buf_len); + task->size = buf_len; + task->frame_number = ww->zstd.num_frames++; + task->ww = ww; + + BLI_mutex_lock(&ww->zstd.mutex); + BLI_addtail(&ww->zstd.tasks, task); + + /* If there's a free worker thread, just push the block into that thread. + * Otherwise, we wait for the earliest thread to finish. + * We look up the earliest thread while holding the mutex, but release it + * before joining the thread to prevent a deadlock. */ + ZstdWriteBlockTask *first_task = ww->zstd.tasks.first; + BLI_mutex_unlock(&ww->zstd.mutex); + if (!BLI_available_threads(&ww->zstd.threadpool)) { + BLI_threadpool_remove(&ww->zstd.threadpool, first_task); + + /* If the task list was empty before we pushed our task, there should + * always be a free thread. */ + BLI_assert(first_task != task); + BLI_remlink(&ww->zstd.tasks, first_task); + MEM_freeN(first_task); + } + BLI_threadpool_insert(&ww->zstd.threadpool, task); + + return buf_len; } -#undef FILE_HANDLE /* --- end compression types --- */ @@ -226,11 +383,11 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww) memset(r_ww, 0, sizeof(*r_ww)); switch (ww_type) { - case WW_WRAP_ZLIB: { - r_ww->open = ww_open_zlib; - r_ww->close = ww_close_zlib; - r_ww->write = ww_write_zlib; - r_ww->use_buf = false; + case WW_WRAP_ZSTD: { + r_ww->open = ww_open_zstd; + r_ww->close = ww_close_zstd; + r_ww->write = ww_write_zstd; + r_ww->use_buf = true; break; } default: { @@ -252,10 +409,17 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww) typedef struct { const struct SDNA *sdna; - /** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */ - uchar *buf; - /** Number of bytes used in #WriteData.buf (flushed when exceeded). */ - size_t buf_used_len; + struct { + /** Use for file and memory writing (size stored in max_size). */ + uchar *buf; + /** Number of bytes used in #WriteData.buf (flushed when exceeded). */ + size_t used_len; + + /** Maximum size of the buffer. */ + size_t max_size; + /** Threshold above which writes get their own chunk. */ + size_t chunk_size; + } buffer; #ifdef USE_WRITE_DATA_LEN /** Total number of bytes written. */ @@ -271,7 +435,7 @@ typedef struct { bool use_memfile; /** - * Wrap writing, so we can use zlib or + * Wrap writing, so we can use zstd or * other compression types later, see: G_FILE_COMPRESS * Will be NULL for UNDO. */ @@ -291,7 +455,15 @@ static WriteData *writedata_new(WriteWrap *ww) wd->ww = ww; if ((ww == NULL) || (ww->use_buf)) { - wd->buf = MEM_mallocN(MYWRITE_BUFFER_SIZE, "wd->buf"); + if (ww == NULL) { + wd->buffer.max_size = MEM_BUFFER_SIZE; + wd->buffer.chunk_size = MEM_CHUNK_SIZE; + } + else { + wd->buffer.max_size = ZSTD_BUFFER_SIZE; + wd->buffer.chunk_size = ZSTD_CHUNK_SIZE; + } + wd->buffer.buf = MEM_mallocN(wd->buffer.max_size, "wd->buffer.buf"); } return wd; @@ -325,8 +497,8 @@ static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen) static void writedata_free(WriteData *wd) { - if (wd->buf) { - MEM_freeN(wd->buf); + if (wd->buffer.buf) { + MEM_freeN(wd->buffer.buf); } MEM_freeN(wd); } @@ -343,9 +515,9 @@ static void writedata_free(WriteData *wd) */ static void mywrite_flush(WriteData *wd) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } } @@ -369,20 +541,20 @@ static void mywrite(WriteData *wd, const void *adr, size_t len) wd->write_len += len; #endif - if (wd->buf == NULL) { + if (wd->buffer.buf == NULL) { writedata_do_write(wd, adr, len); } else { /* if we have a single big chunk, write existing data in * buffer and write out big chunk in smaller pieces */ - if (len > MYWRITE_MAX_CHUNK) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (len > wd->buffer.chunk_size) { + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } do { - size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK); + size_t writelen = MIN2(len, wd->buffer.chunk_size); writedata_do_write(wd, adr, writelen); adr = (const char *)adr + writelen; len -= writelen; @@ -392,14 +564,14 @@ static void mywrite(WriteData *wd, const void *adr, size_t len) } /* if data would overflow buffer, write out the buffer */ - if (len + wd->buf_used_len > MYWRITE_BUFFER_SIZE - 1) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (len + wd->buffer.used_len > wd->buffer.max_size - 1) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } /* append data at end of buffer */ - memcpy(&wd->buf[wd->buf_used_len], adr, len); - wd->buf_used_len += len; + memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len); + wd->buffer.used_len += len; } } @@ -430,9 +602,9 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren */ static bool mywrite_end(WriteData *wd) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } if (wd->use_memfile) { @@ -1150,7 +1322,6 @@ bool BLO_write_file(Main *mainvar, ReportList *reports) { char tempname[FILE_MAX + 1]; - eWriteWrapType ww_type; WriteWrap ww; eBLO_WritePathRemap remap_mode = params->remap_mode; @@ -1172,14 +1343,7 @@ bool BLO_write_file(Main *mainvar, /* open temporary file, so we preserve the original in case we crash */ BLI_snprintf(tempname, sizeof(tempname), "%s@", filepath); - if (write_flags & G_FILE_COMPRESS) { - ww_type = WW_WRAP_ZLIB; - } - else { - ww_type = WW_WRAP_NONE; - } - - ww_handle_init(ww_type, &ww); + ww_handle_init((write_flags & G_FILE_COMPRESS) ? WW_WRAP_ZSTD : WW_WRAP_NONE, &ww); if (ww.open(&ww, tempname) == false) { BKE_reportf( diff --git a/source/blender/draw/engines/eevee/eevee_lookdev.c b/source/blender/draw/engines/eevee/eevee_lookdev.c index f4dc553e982..879a7b08eba 100644 --- a/source/blender/draw/engines/eevee/eevee_lookdev.c +++ b/source/blender/draw/engines/eevee/eevee_lookdev.c @@ -246,7 +246,7 @@ void EEVEE_lookdev_cache_init(EEVEE_Data *vedata, DRW_shgroup_uniform_float_copy(grp, "studioLightIntensity", shading->studiolight_intensity); BKE_studiolight_ensure_flag(sl, STUDIOLIGHT_EQUIRECT_RADIANCE_GPUTEXTURE); DRW_shgroup_uniform_texture_ex(grp, "studioLight", sl->equirect_radiance_gputexture, state); - /* Do not fadeout when doing probe rendering, only when drawing the background */ + /* Do not fade-out when doing probe rendering, only when drawing the background. */ DRW_shgroup_uniform_float_copy(grp, "backgroundAlpha", 1.0f); } else { diff --git a/source/blender/editors/gpencil/gpencil_interpolate.c b/source/blender/editors/gpencil/gpencil_interpolate.c index 8640ffa67cf..a8bd3b11bb1 100644 --- a/source/blender/editors/gpencil/gpencil_interpolate.c +++ b/source/blender/editors/gpencil/gpencil_interpolate.c @@ -890,9 +890,9 @@ static int gpencil_interpolate_modal(bContext *C, wmOperator *op, const wmEvent } case MOUSEMOVE: /* calculate new position */ { - /* only handle mousemove if not doing numinput */ + /* Only handle mouse-move if not doing numeric-input. */ if (has_numinput == false) { - /* update shift based on position of mouse */ + /* Update shift based on position of mouse. */ gpencil_mouse_update_shift(tgpi, op, event); /* update screen */ diff --git a/source/blender/editors/gpencil/gpencil_primitive.c b/source/blender/editors/gpencil/gpencil_primitive.c index 894fb83d70e..5ecb6d9a212 100644 --- a/source/blender/editors/gpencil/gpencil_primitive.c +++ b/source/blender/editors/gpencil/gpencil_primitive.c @@ -1944,9 +1944,9 @@ static int gpencil_primitive_modal(bContext *C, wmOperator *op, const wmEvent *e if (ELEM(tgpi->flag, IN_CURVE_EDIT)) { break; } - /* only handle mousemove if not doing numinput */ + /* Only handle mouse-move if not doing numeric-input. */ if (has_numinput == false) { - /* update position of mouse */ + /* Update position of mouse. */ copy_v2_v2(tgpi->end, tgpi->mval); copy_v2_v2(tgpi->start, tgpi->origin); if (tgpi->flag == IDLE) { diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c index b8f324cba83..76f6640c714 100644 --- a/source/blender/editors/interface/interface_handlers.c +++ b/source/blender/editors/interface/interface_handlers.c @@ -6025,7 +6025,7 @@ static int ui_do_but_BLOCK(bContext *C, uiBut *but, uiHandleButtonData *data, co * the slot menu fails to switch a second time. * * The active state of the button could be maintained some other way - * and remove this mousemove event. + * and remove this mouse-move event. */ WM_event_add_mousemove(data->window); @@ -8364,7 +8364,7 @@ static void button_activate_state(bContext *C, uiBut *but, uiHandleButtonState s } } - /* wait for mousemove to enable drag */ + /* Wait for mouse-move to enable drag. */ if (state == BUTTON_STATE_WAIT_DRAG) { but->flag &= ~UI_SELECT; } @@ -8631,9 +8631,9 @@ static void button_activate_exit( ui_but_update(but); } - /* adds empty mousemove in queue for re-init handler, in case mouse is + /* Adds empty mouse-move in queue for re-initialize handler, in case mouse is * still over a button. We cannot just check for this ourselves because - * at this point the mouse may be over a button in another region */ + * at this point the mouse may be over a button in another region. */ if (mousemove) { WM_event_add_mousemove(CTX_wm_window(C)); } diff --git a/source/blender/editors/interface/interface_templates.c b/source/blender/editors/interface/interface_templates.c index 351b73c320b..0e5a6a79137 100644 --- a/source/blender/editors/interface/interface_templates.c +++ b/source/blender/editors/interface/interface_templates.c @@ -6489,6 +6489,17 @@ void uiTemplateCacheFile(uiLayout *layout, uiLayoutSetActive(row, engine_supports_procedural); uiItemR(row, &fileptr, "use_render_procedural", 0, NULL, ICON_NONE); + const bool use_render_procedural = RNA_boolean_get(&fileptr, "use_render_procedural"); + const bool use_prefetch = RNA_boolean_get(&fileptr, "use_prefetch"); + + row = uiLayoutRow(layout, false); + uiLayoutSetEnabled(row, use_render_procedural); + uiItemR(row, &fileptr, "use_prefetch", 0, NULL, ICON_NONE); + + sub = uiLayoutRow(layout, false); + uiLayoutSetEnabled(sub, use_prefetch && use_render_procedural); + uiItemR(sub, &fileptr, "prefetch_cache_size", 0, NULL, ICON_NONE); + row = uiLayoutRowWithHeading(layout, true, IFACE_("Override Frame")); sub = uiLayoutRow(row, true); uiLayoutSetPropDecorate(sub, false); diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c index b8bbf2d3e70..956658bd2b7 100644 --- a/source/blender/editors/mesh/editmesh_tools.c +++ b/source/blender/editors/mesh/editmesh_tools.c @@ -8627,7 +8627,7 @@ static int edbm_point_normals_modal(bContext *C, wmOperator *op, const wmEvent * RNA_enum_set(op->ptr, "mode", mode); } - /* Only handle mousemove event in case we are in mouse mode. */ + /* Only handle mouse-move event in case we are in mouse mode. */ if (event->type == MOUSEMOVE || force_mousemove) { if (mode == EDBM_CLNOR_POINTTO_MODE_MOUSE) { ARegion *region = CTX_wm_region(C); diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c index 85883a2d29a..8afc5c583e0 100644 --- a/source/blender/editors/physics/particle_edit.c +++ b/source/blender/editors/physics/particle_edit.c @@ -4667,7 +4667,7 @@ typedef struct BrushEdit { int lastmouse[2]; float zfac; - /* optional cached view settings to avoid setting on every mousemove */ + /** Optional cached view settings to avoid setting on every mouse-move. */ PEData data; } BrushEdit; diff --git a/source/blender/editors/render/render_update.c b/source/blender/editors/render/render_update.c index 6db148eb4e1..8bc2281db73 100644 --- a/source/blender/editors/render/render_update.c +++ b/source/blender/editors/render/render_update.c @@ -64,7 +64,9 @@ #include <stdio.h> -/***************************** Render Engines ********************************/ +/* -------------------------------------------------------------------- */ +/** \name Render Engines + * \{ */ /* Update 3D viewport render or draw engine on changes to the scene or view settings. */ void ED_render_view3d_update(Depsgraph *depsgraph, @@ -206,15 +208,15 @@ void ED_render_engine_changed(Main *bmain, const bool update_scene_data) } } - /* Update CacheFiles to ensure that procedurals are properly taken into account. */ + /* Update #CacheFiles to ensure that procedurals are properly taken into account. */ LISTBASE_FOREACH (CacheFile *, cachefile, &bmain->cachefiles) { - /* Only update cachefiles which are set to use a render procedural. We do not use - * BKE_cachefile_uses_render_procedural here as we need to update regardless of the current - * engine or its settings. */ + /* Only update cache-files which are set to use a render procedural. + * We do not use #BKE_cachefile_uses_render_procedural here as we need to update regardless of + * the current engine or its settings. */ if (cachefile->use_render_procedural) { DEG_id_tag_update(&cachefile->id, ID_RECALC_COPY_ON_WRITE); - /* Rebuild relations so that modifiers are reconnected to or disconnected from the cachefile. - */ + /* Rebuild relations so that modifiers are reconnected to or disconnected from the + * cache-file. */ DEG_relations_tag_update(bmain); } } @@ -227,10 +229,16 @@ void ED_render_view_layer_changed(Main *bmain, bScreen *screen) } } -/***************************** Updates *********************************** - * ED_render_id_flush_update gets called from DEG_id_tag_update, to do * - * editor level updates when the ID changes. when these ID blocks are in * - * the dependency graph, we can get rid of the manual dependency checks. */ +/** \} */ + +/* -------------------------------------------------------------------- */ +/** \name Updates + * + * #ED_render_id_flush_update gets called from #DEG_id_tag_update, + * to do editor level updates when the ID changes. + * When these ID blocks are in the dependency graph, + * we can get rid of the manual dependency checks. + * \{ */ static void material_changed(Main *UNUSED(bmain), Material *ma) { @@ -336,3 +344,5 @@ void ED_render_id_flush_update(const DEGEditorUpdateContext *update_ctx, ID *id) break; } } + +/** \} */ diff --git a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c index 49299d73337..edc34d0d883 100644 --- a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c +++ b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c @@ -299,7 +299,9 @@ static void view3d_ruler_item_project(RulerInfo *ruler_info, float r_co[3], cons ED_view3d_win_to_3d_int(ruler_info->area->spacedata.first, ruler_info->region, r_co, xy, r_co); } -/* use for mousemove events */ +/** + * Use for mouse-move events. + */ static bool view3d_ruler_item_mousemove(struct Depsgraph *depsgraph, RulerInfo *ruler_info, RulerItem *ruler_item, diff --git a/source/blender/editors/transform/transform_convert_action.c b/source/blender/editors/transform/transform_convert_action.c index 8a3b254be5c..a5565b5fb88 100644 --- a/source/blender/editors/transform/transform_convert_action.c +++ b/source/blender/editors/transform/transform_convert_action.c @@ -241,16 +241,15 @@ static int GPLayerToTransData(TransData *td, for (gpf = gpl->frames.first; gpf; gpf = gpf->next) { if (is_prop_edit || (gpf->flag & GP_FRAME_SELECT)) { if (FrameOnMouseSide(side, (float)gpf->framenum, cfra)) { - /* memory is calloc'ed, so that should zero everything nicely for us */ - td->val = &tfd->val; - td->ival = (float)gpf->framenum; + tfd->val = (float)gpf->framenum; + tfd->sdata = &gpf->framenum; + + td->val = td->loc = &tfd->val; /* XXX: It's not a 3d array. */ + td->ival = td->iloc[0] = (float)gpf->framenum; td->center[0] = td->ival; td->center[1] = ypos; - tfd->val = (float)gpf->framenum; - tfd->sdata = &gpf->framenum; - /* Advance `td` now. */ td++; tfd++; diff --git a/source/blender/editors/transform/transform_convert_armature.c b/source/blender/editors/transform/transform_convert_armature.c index f56d60b7376..5627a910ab4 100644 --- a/source/blender/editors/transform/transform_convert_armature.c +++ b/source/blender/editors/transform/transform_convert_armature.c @@ -131,12 +131,12 @@ static void autokeyframe_pose( ListBase dsources = {NULL, NULL}; - /* add datasource override for the camera object */ + /* Add data-source override for the camera object. */ ANIM_relative_keyingset_add_source(&dsources, id, &RNA_PoseBone, pchan); /* only insert into active keyingset? */ if (IS_AUTOKEY_FLAG(scene, ONLYKEYINGSET) && (active_ks)) { - /* run the active Keying Set on the current datasource */ + /* Run the active Keying Set on the current data-source. */ ANIM_apply_keyingset( C, &dsources, NULL, active_ks, MODIFYKEY_MODE_INSERT, anim_eval_context.eval_time); } diff --git a/source/blender/editors/transform/transform_convert_object.c b/source/blender/editors/transform/transform_convert_object.c index ee6cb391fdc..bcbac009948 100644 --- a/source/blender/editors/transform/transform_convert_object.c +++ b/source/blender/editors/transform/transform_convert_object.c @@ -749,7 +749,7 @@ static void autokeyframe_object( /* Get flags used for inserting keyframes. */ flag = ANIM_get_keyframing_flags(scene, true); - /* add datasource override for the object */ + /* Add data-source override for the object. */ ANIM_relative_keyingset_add_source(&dsources, id, NULL, NULL); if (IS_AUTOKEY_FLAG(scene, ONLYKEYINGSET) && (active_ks)) { diff --git a/source/blender/editors/transform/transform_snap_animation.c b/source/blender/editors/transform/transform_snap_animation.c index be7fc65a6c2..08335924ddf 100644 --- a/source/blender/editors/transform/transform_snap_animation.c +++ b/source/blender/editors/transform/transform_snap_animation.c @@ -35,14 +35,14 @@ #include "transform_snap.h" /* -------------------------------------------------------------------- */ -/** \name Snappint in Anim Editors +/** \name Snapping in Anim Editors * \{ */ /** * This function returns the snapping 'mode' for Animation Editors only. * We cannot use the standard snapping due to NLA-strip scaling complexities. * - * TODO: these modifier checks should be key-mappable. + * TODO: these modifier checks should be accessible from the key-map. */ short getAnimEdit_SnapMode(TransInfo *t) { diff --git a/source/blender/io/collada/collada_internal.cpp b/source/blender/io/collada/collada_internal.cpp index 355aa5c22f0..bd6f496c8ec 100644 --- a/source/blender/io/collada/collada_internal.cpp +++ b/source/blender/io/collada/collada_internal.cpp @@ -162,7 +162,7 @@ void UnitConverter::calculate_scale(Scene &sce) * Translation map. * Used to translate every COLLADA id to a valid id, no matter what "wrong" letters may be * included. Look at the IDREF XSD declaration for more. - * Follows strictly the COLLADA XSD declaration which explicitly allows non-english chars, + * Follows strictly the COLLADA XSD declaration which explicitly allows non-English chars, * like special chars (e.g. micro sign), umlauts and so on. * The COLLADA spec also allows additional chars for member access ('.'), these * must obviously be removed too, otherwise they would be heavily misinterpreted. diff --git a/source/blender/makesdna/DNA_cachefile_defaults.h b/source/blender/makesdna/DNA_cachefile_defaults.h index 521b72567d4..74fbe5012ab 100644 --- a/source/blender/makesdna/DNA_cachefile_defaults.h +++ b/source/blender/makesdna/DNA_cachefile_defaults.h @@ -40,6 +40,8 @@ .handle = NULL, \ .handle_filepath[0] = '\0', \ .handle_readers = NULL, \ + .use_prefetch = 1, \ + .prefetch_cache_size = 4096, \ } /** \} */ diff --git a/source/blender/makesdna/DNA_cachefile_types.h b/source/blender/makesdna/DNA_cachefile_types.h index ae4ade49be1..0f4c53a6e7e 100644 --- a/source/blender/makesdna/DNA_cachefile_types.h +++ b/source/blender/makesdna/DNA_cachefile_types.h @@ -101,7 +101,15 @@ typedef struct CacheFile { */ char use_render_procedural; - char _pad1[7]; + char _pad1[3]; + + /** Enable data prefetching when using the Cycles Procedural. */ + char use_prefetch; + + /** Size in megabytes for the prefetch cache used by the Cycles Procedural. */ + int prefetch_cache_size; + + char _pad2[7]; char velocity_unit; /* Name of the velocity property in the archive. */ diff --git a/source/blender/makesrna/intern/rna_cachefile.c b/source/blender/makesrna/intern/rna_cachefile.c index cb0a490417d..74d924b8937 100644 --- a/source/blender/makesrna/intern/rna_cachefile.c +++ b/source/blender/makesrna/intern/rna_cachefile.c @@ -150,6 +150,23 @@ static void rna_def_cachefile(BlenderRNA *brna) "determine which file to use in a file sequence"); RNA_def_property_update(prop, 0, "rna_CacheFile_update"); + /* ----------------- Cache controls ----------------- */ + + prop = RNA_def_property(srna, "use_prefetch", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_ui_text( + prop, + "Use Prefetch", + "When enabled, the Cycles Procedural will preload animation data for faster updates"); + RNA_def_property_update(prop, 0, "rna_CacheFile_update"); + + prop = RNA_def_property(srna, "prefetch_cache_size", PROP_INT, PROP_UNSIGNED); + RNA_def_property_ui_text( + prop, + "Prefetch Cache Size", + "Memory usage limit in megabytes for the Cycles Procedural cache, if the data does not " + "fit within the limit, rendering is aborted"); + RNA_def_property_update(prop, 0, "rna_CacheFile_update"); + /* ----------------- Axis Conversion ----------------- */ prop = RNA_def_property(srna, "forward_axis", PROP_ENUM, PROP_NONE); diff --git a/source/blender/modifiers/intern/MOD_subsurf.c b/source/blender/modifiers/intern/MOD_subsurf.c index ce427281db3..db0b769684e 100644 --- a/source/blender/modifiers/intern/MOD_subsurf.c +++ b/source/blender/modifiers/intern/MOD_subsurf.c @@ -358,13 +358,8 @@ static bool get_show_adaptive_options(const bContext *C, Panel *panel) /* Don't show adaptive options if the cycles experimental feature set is disabled. */ Scene *scene = CTX_data_scene(C); - PointerRNA scene_ptr; - RNA_id_pointer_create(&scene->id, &scene_ptr); - if (BKE_scene_uses_cycles(scene)) { - PointerRNA cycles_ptr = RNA_pointer_get(&scene_ptr, "cycles"); - if (RNA_enum_get(&cycles_ptr, "feature_set") != 1) { /* EXPERIMENTAL */ - return false; - } + if (!BKE_scene_uses_cycles_experimental_features(scene)) { + return false; } return true; diff --git a/source/blender/python/intern/bpy_app_icons.c b/source/blender/python/intern/bpy_app_icons.c index 7cca3ae4700..acd809fb8d5 100644 --- a/source/blender/python/intern/bpy_app_icons.c +++ b/source/blender/python/intern/bpy_app_icons.c @@ -35,7 +35,7 @@ /* We may want to load direct from file. */ PyDoc_STRVAR( bpy_app_icons_new_triangles_doc, - ".. function:: new_triangles(range, coords, colors)" + ".. function:: new_triangles(range, coords, colors)\n" "\n" " Create a new icon from triangle geometry.\n" "\n" @@ -91,7 +91,7 @@ static PyObject *bpy_app_icons_new_triangles(PyObject *UNUSED(self), PyObject *a } PyDoc_STRVAR(bpy_app_icons_new_triangles_from_file_doc, - ".. function:: new_triangles_from_file(filename)" + ".. function:: new_triangles_from_file(filename)\n" "\n" " Create a new icon from triangle geometry.\n" "\n" @@ -122,7 +122,7 @@ static PyObject *bpy_app_icons_new_triangles_from_file(PyObject *UNUSED(self), } PyDoc_STRVAR(bpy_app_icons_release_doc, - ".. function:: release(icon_id)" + ".. function:: release(icon_id)\n" "\n" " Release the icon.\n"); static PyObject *bpy_app_icons_release(PyObject *UNUSED(self), PyObject *args, PyObject *kw) diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c index 604c9900355..86bd840ce31 100644 --- a/source/blender/sequencer/intern/image_cache.c +++ b/source/blender/sequencer/intern/image_cache.c @@ -102,7 +102,7 @@ /* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */ #define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf" #define DCACHE_IMAGES_PER_FILE 100 -#define DCACHE_CURRENT_VERSION 1 +#define DCACHE_CURRENT_VERSION 2 #define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */ typedef struct DiskCacheHeaderEntry { @@ -496,24 +496,34 @@ static size_t deflate_imbuf_to_file(ImBuf *ibuf, int level, DiskCacheHeaderEntry *header_entry) { - if (ibuf->rect) { - return BLI_gzip_mem_to_file_at_pos( - ibuf->rect, header_entry->size_raw, file, header_entry->offset, level); + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + + /* Apply compression if wanted, otherwise just write directly to the file. */ + if (level > 0) { + return BLI_file_zstd_from_mem_at_pos( + data, header_entry->size_raw, file, header_entry->offset, level); } - return BLI_gzip_mem_to_file_at_pos( - ibuf->rect_float, header_entry->size_raw, file, header_entry->offset, level); + fseek(file, header_entry->offset, SEEK_SET); + return fwrite(data, 1, header_entry->size_raw, file); } static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry) { - if (ibuf->rect) { - return BLI_ungzip_file_to_mem_at_pos( - ibuf->rect, header_entry->size_raw, file, header_entry->offset); + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + char header[4]; + fseek(file, header_entry->offset, SEEK_SET); + if (fread(header, 1, sizeof(header), file) != sizeof(header)) { + return 0; + } + + /* Check if the data is compressed or raw. */ + if (BLI_file_magic_is_zstd(header)) { + return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset); } - return BLI_ungzip_file_to_mem_at_pos( - ibuf->rect_float, header_entry->size_raw, file, header_entry->offset); + fseek(file, header_entry->offset, SEEK_SET); + return fread(data, 1, header_entry->size_raw, file); } static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header) diff --git a/source/blender/windowmanager/CMakeLists.txt b/source/blender/windowmanager/CMakeLists.txt index b7fbb9bb82b..4d65726fe2b 100644 --- a/source/blender/windowmanager/CMakeLists.txt +++ b/source/blender/windowmanager/CMakeLists.txt @@ -48,10 +48,6 @@ set(INC ${CMAKE_BINARY_DIR}/source/blender/makesdna/intern ) -set(INC_SYS - ${ZLIB_INCLUDE_DIRS} -) - set(SRC intern/wm.c intern/wm_cursors.c diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c index 5cc361fc1bd..b9a3dd0c3fb 100644 --- a/source/blender/windowmanager/intern/wm_event_system.c +++ b/source/blender/windowmanager/intern/wm_event_system.c @@ -3471,8 +3471,8 @@ void wm_event_do_handlers(bContext *C) } CTX_wm_area_set(C, NULL); - /* NOTE: do not escape on WM_HANDLER_BREAK, - * mousemove needs handled for previous area. */ + /* NOTE: do not escape on #WM_HANDLER_BREAK, + * mouse-move needs handled for previous area. */ } } diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c index b53ad0ee927..f83511e76f0 100644 --- a/source/blender/windowmanager/intern/wm_files.c +++ b/source/blender/windowmanager/intern/wm_files.c @@ -28,11 +28,10 @@ * winsock stuff. */ #include <errno.h> +#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */ #include <stddef.h> #include <string.h> -#include "zlib.h" /* wm_read_exotic() */ - #ifdef WIN32 /* Need to include windows.h so _WIN32_IE is defined. */ # include <windows.h> @@ -51,6 +50,7 @@ #include "BLI_blenlib.h" #include "BLI_fileops_types.h" +#include "BLI_filereader.h" #include "BLI_linklist.h" #include "BLI_math.h" #include "BLI_system.h" @@ -481,53 +481,64 @@ static void wm_init_userdef(Main *bmain) /* intended to check for non-blender formats but for now it only reads blends */ static int wm_read_exotic(const char *name) { - int len; - gzFile gzfile; + /* make sure we're not trying to read a directory.... */ + + int namelen = strlen(name); + if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) { + return BKE_READ_EXOTIC_FAIL_PATH; + } + + /* open the file. */ + const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0); + if (filedes == -1) { + return BKE_READ_EXOTIC_FAIL_OPEN; + } + + FileReader *rawfile = BLI_filereader_new_file(filedes); + if (rawfile == NULL) { + return BKE_READ_EXOTIC_FAIL_OPEN; + } + + /* read the header (7 bytes are enough to identify all known types). */ char header[7]; - int retval; + if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) { + rawfile->close(rawfile); + return BKE_READ_EXOTIC_FAIL_FORMAT; + } + rawfile->seek(rawfile, 0, SEEK_SET); - /* make sure we're not trying to read a directory.... */ + /* check for uncompressed .blend */ + if (STREQLEN(header, "BLENDER", 7)) { + rawfile->close(rawfile); + return BKE_READ_EXOTIC_OK_BLEND; + } - len = strlen(name); - if (len > 0 && ELEM(name[len - 1], '/', '\\')) { - retval = BKE_READ_EXOTIC_FAIL_PATH; + /* check for compressed .blend */ + FileReader *compressed_file = NULL; + if (BLI_file_magic_is_gzip(header)) { + /* In earlier versions of Blender (before 3.0), compressed files used Gzip instead of Zstd. + * While these files will no longer be written, there still needs to be reading support. */ + compressed_file = BLI_filereader_new_gzip(rawfile); + } + else if (BLI_file_magic_is_zstd(header)) { + compressed_file = BLI_filereader_new_zstd(rawfile); } - else { - gzfile = BLI_gzopen(name, "rb"); - if (gzfile == NULL) { - retval = BKE_READ_EXOTIC_FAIL_OPEN; - } - else { - len = gzread(gzfile, header, sizeof(header)); - gzclose(gzfile); - if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) { - retval = BKE_READ_EXOTIC_OK_BLEND; - } - else { - /* We may want to support loading other file formats - * from their header bytes or file extension. - * This used to be supported in the code below and may be added - * back at some point. */ -#if 0 - WM_cursor_wait(true); - if (is_foo_format(name)) { - read_foo(name); - retval = BKE_READ_EXOTIC_OK_OTHER; - } - else -#endif - { - retval = BKE_READ_EXOTIC_FAIL_FORMAT; - } -#if 0 - WM_cursor_wait(false); -#endif - } + /* If a compression signature matches, try decompressing the start and check if it's a .blend */ + if (compressed_file != NULL) { + size_t len = compressed_file->read(compressed_file, header, sizeof(header)); + compressed_file->close(compressed_file); + if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) { + return BKE_READ_EXOTIC_OK_BLEND; } } + else { + rawfile->close(rawfile); + } + + /* Add check for future file formats here. */ - return retval; + return BKE_READ_EXOTIC_FAIL_FORMAT; } /** \} */ |