From 2b170f16d6ded9b3bcb428121b27274ae8637555 Mon Sep 17 00:00:00 2001 From: Lukas Stockner Date: Thu, 19 Aug 2021 23:57:00 +0200 Subject: Refactor low-level blendfile reading into separate files Instead of handling mmap, compression etc. all directly in readfile.c, refactor the code to use a generic FileReader. This makes it easier to add new compression methods or similar, and allows to reuse the logic in other places (e.g. thumbnail reading). Reviewed By: campbellbarton, brecht, mont29 Differential Revision: https://developer.blender.org/D5799 --- source/blender/blenlib/BLI_fileops.h | 2 + source/blender/blenlib/BLI_filereader.h | 79 ++++ source/blender/blenlib/CMakeLists.txt | 4 + source/blender/blenlib/intern/fileops.c | 7 + source/blender/blenlib/intern/filereader_file.c | 80 ++++ source/blender/blenlib/intern/filereader_gzip.c | 108 +++++ source/blender/blenlib/intern/filereader_memory.c | 145 +++++++ source/blender/blenloader/BLO_undofile.h | 14 + source/blender/blenloader/intern/readfile.c | 449 ++++----------------- source/blender/blenloader/intern/readfile.h | 32 +- source/blender/blenloader/intern/undofile.c | 95 +++++ source/blender/blenloader/intern/versioning_250.c | 3 +- .../blender/blenloader/intern/versioning_legacy.c | 3 +- source/blender/blenloader/intern/writefile.c | 2 +- source/blender/windowmanager/intern/wm_files.c | 88 ++-- 15 files changed, 667 insertions(+), 444 deletions(-) create mode 100644 source/blender/blenlib/BLI_filereader.h create mode 100644 source/blender/blenlib/intern/filereader_file.c create mode 100644 source/blender/blenlib/intern/filereader_gzip.c create mode 100644 source/blender/blenlib/intern/filereader_memory.c diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h index 7cfecc798a7..12fa73279c8 100644 --- a/source/blender/blenlib/BLI_fileops.h +++ b/source/blender/blenlib/BLI_fileops.h @@ -166,6 +166,8 @@ size_t BLI_gzip_mem_to_file_at_pos(void *buf, int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +bool BLI_file_magic_is_gzip(const char header[4]); + size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT; size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); diff --git a/source/blender/blenlib/BLI_filereader.h b/source/blender/blenlib/BLI_filereader.h new file mode 100644 index 00000000000..dbcaa7cb1b6 --- /dev/null +++ b/source/blender/blenlib/BLI_filereader.h @@ -0,0 +1,79 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2001-2002 by NaN Holding BV. + * All rights reserved. + */ + +/** \file + * \ingroup bli + * \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...). + */ + +#pragma once + +#ifdef WIN32 +# include "BLI_winstuff.h" +#else +# include +#endif + +#include "BLI_compiler_attrs.h" +#include "BLI_utildefines.h" + +#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__) +typedef int64_t off64_t; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct FileReader; + +typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size); +typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence); +typedef void (*FileReaderCloseFn)(struct FileReader *reader); + +/* General structure for all FileReaders, implementations add custom fields at the end. */ +typedef struct FileReader { + FileReaderReadFn read; + FileReaderSeekFn seek; + FileReaderCloseFn close; + + off64_t offset; +} FileReader; + +/* Functions for opening the various types of FileReader. + * They either succeed and return a valid FileReader, or fail and return NULL. + * + * If a FileReader is created, it has to be cleaned up and freed by calling + * its close() function unless another FileReader has taken ownership - for example, + * Gzip takes over the base FileReader and will clean it up when their clean() is called. + */ + +/* Create FileReader from raw file descriptor. */ +FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT; +/* Create FileReader from raw file descriptor using memory-mapped IO. */ +FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT; +/* Create FileReader from a region of memory. */ +FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT + ATTR_NONNULL(); +/* Create FileReader from applying Gzip decompression on an underlying file. */ +FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); + +#ifdef __cplusplus +} +#endif diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt index b6603dce378..d2ba9e74c90 100644 --- a/source/blender/blenlib/CMakeLists.txt +++ b/source/blender/blenlib/CMakeLists.txt @@ -75,6 +75,9 @@ set(SRC intern/endian_switch.c intern/expr_pylike_eval.c intern/fileops.c + intern/filereader_file.c + intern/filereader_gzip.c + intern/filereader_memory.c intern/fnmatch.c intern/freetypefont.c intern/gsqueue.c @@ -194,6 +197,7 @@ set(SRC BLI_enumerable_thread_specific.hh BLI_expr_pylike_eval.h BLI_fileops.h + BLI_filereader.h BLI_fileops_types.h BLI_float2.hh BLI_float3.hh diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c index ac034d2b5cd..6fc2222241b 100644 --- a/source/blender/blenlib/intern/fileops.c +++ b/source/blender/blenlib/intern/fileops.c @@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g #undef CHUNK +bool BLI_file_magic_is_gzip(const char header[4]) +{ + /* GZIP itself starts with the magic bytes 0x1f 0x8b. + * The third byte indicates the compression method, which is 0x08 for DEFLATE. */ + return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08; +} + /** * Returns true if the file with the specified name can be written. * This implementation uses access(2), which makes the check according diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c new file mode 100644 index 00000000000..3a833871e27 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_file.c @@ -0,0 +1,80 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#ifndef WIN32 +# include /* for read close */ +#else +# include "BLI_winstuff.h" +# include "winsock2.h" +# include /* for open close read */ +#endif + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + int filedes; +} RawFileReader; + +static ssize_t file_read(FileReader *reader, void *buffer, size_t size) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + ssize_t readsize = read(rawfile->filedes, buffer, size); + + if (readsize >= 0) { + rawfile->reader.offset += readsize; + } + + return readsize; +} + +static off64_t file_seek(FileReader *reader, off64_t offset, int whence) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence); + return rawfile->reader.offset; +} + +static void file_close(FileReader *reader) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + close(rawfile->filedes); + MEM_freeN(rawfile); +} + +FileReader *BLI_filereader_new_file(int filedes) +{ + RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__); + + rawfile->filedes = filedes; + + rawfile->reader.read = file_read; + rawfile->reader.seek = file_seek; + rawfile->reader.close = file_close; + + return (FileReader *)rawfile; +} diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c new file mode 100644 index 00000000000..72eb153a8b9 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_gzip.c @@ -0,0 +1,108 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + FileReader *base; + + z_stream strm; + + void *in_buf; + size_t in_size; +} GzipReader; + +static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size) +{ + GzipReader *gzip = (GzipReader *)reader; + + gzip->strm.avail_out = size; + gzip->strm.next_out = buffer; + + while (gzip->strm.avail_out > 0) { + if (gzip->strm.avail_in == 0) { + /* Ran out of buffered input data, read some more. */ + size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size); + + if (readsize > 0) { + /* We got some data, so mark the buffer as refilled. */ + gzip->strm.avail_in = readsize; + gzip->strm.next_in = gzip->in_buf; + } + else { + /* The underlying file is EOF, so return as much as we can. */ + break; + } + } + + int ret = inflate(&gzip->strm, Z_NO_FLUSH); + + if (ret != Z_OK && ret != Z_BUF_ERROR) { + break; + } + } + + ssize_t read_len = size - gzip->strm.avail_out; + gzip->reader.offset += read_len; + return read_len; +} + +static void gzip_close(FileReader *reader) +{ + GzipReader *gzip = (GzipReader *)reader; + + if (inflateEnd(&gzip->strm) != Z_OK) { + printf("close gzip stream error\n"); + } + MEM_freeN((void *)gzip->in_buf); + + gzip->base->close(gzip->base); + MEM_freeN(gzip); +} + +FileReader *BLI_filereader_new_gzip(FileReader *base) +{ + GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__); + gzip->base = base; + + if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) { + MEM_freeN(gzip); + return NULL; + } + + gzip->in_size = 256 * 2014; + gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf"); + + gzip->reader.read = gzip_read; + gzip->reader.seek = NULL; + gzip->reader.close = gzip_close; + + return (FileReader *)gzip; +} diff --git a/source/blender/blenlib/intern/filereader_memory.c b/source/blender/blenlib/intern/filereader_memory.c new file mode 100644 index 00000000000..150fed7d1cc --- /dev/null +++ b/source/blender/blenlib/intern/filereader_memory.c @@ -0,0 +1,145 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" +#include "BLI_mmap.h" + +#include "MEM_guardedalloc.h" + +/* This file implements both memory-backed and memory-mapped-file-backed reading. */ +typedef struct { + FileReader reader; + + const char *data; + BLI_mmap_file *mmap; + size_t length; +} MemoryReader; + +static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + memcpy(buffer, mem->data + mem->reader.offset, readsize); + mem->reader.offset += readsize; + + return readsize; +} + +static off64_t memory_seek(FileReader *reader, off64_t offset, int whence) +{ + MemoryReader *mem = (MemoryReader *)reader; + + off64_t new_pos; + if (whence == SEEK_CUR) { + new_pos = mem->reader.offset + offset; + } + else if (whence == SEEK_SET) { + new_pos = offset; + } + else if (whence == SEEK_END) { + new_pos = mem->length + offset; + } + else { + return -1; + } + + if (new_pos < 0 || new_pos > mem->length) { + return -1; + } + + mem->reader.offset = new_pos; + return mem->reader.offset; +} + +static void memory_close_raw(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLI_filereader_new_memory(const void *data, size_t len) +{ + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->data = (const char *)data; + mem->length = len; + + mem->reader.read = memory_read_raw; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_raw; + + return (FileReader *)mem; +} + +/* Memory-mapped file reading. + * By using `mmap()`, we can map a file so that it can be treated like normal memory, + * meaning that we can just read from it with `memcpy()` etc. + * This avoids system call overhead and can significantly speed up file loading. + */ + +static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) { + return 0; + } + + mem->reader.offset += readsize; + + return readsize; +} + +static void memory_close_mmap(FileReader *reader) +{ + MemoryReader *mem = (MemoryReader *)reader; + BLI_mmap_free(mem->mmap); + MEM_freeN(mem); +} + +FileReader *BLI_filereader_new_mmap(int filedes) +{ + BLI_mmap_file *mmap = BLI_mmap_open(filedes); + if (mmap == NULL) { + return NULL; + } + + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->mmap = mmap; + mem->length = BLI_lseek(filedes, 0, SEEK_END); + + mem->reader.read = memory_read_mmap; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_mmap; + + return (FileReader *)mem; +} diff --git a/source/blender/blenloader/BLO_undofile.h b/source/blender/blenloader/BLO_undofile.h index fc41a6e832f..4e240e2462b 100644 --- a/source/blender/blenloader/BLO_undofile.h +++ b/source/blender/blenloader/BLO_undofile.h @@ -24,6 +24,8 @@ * \ingroup blenloader */ +#include "BLI_filereader.h" + struct GHash; struct Scene; @@ -65,6 +67,16 @@ typedef struct MemFileUndoData { size_t undo_size; } MemFileUndoData; +/* FileReader-compatible wrapper for reading MemFiles */ +typedef struct { + FileReader reader; + + MemFile *memfile; + int undo_direction; + + bool memchunk_identical; +} UndoReader; + /* actually only used writefile.c */ void BLO_memfile_write_init(MemFileWriteData *mem_data, @@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile, struct Main *bmain, struct Scene **r_scene); extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename); + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction); diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index e48c305fc4b..82fbb2083df 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -21,8 +21,6 @@ * \ingroup blenloader */ -#include "zlib.h" - #include /* for isdigit. */ #include /* for open flags (O_BINARY, O_RDONLY). */ #include @@ -71,7 +69,6 @@ #include "BLI_math.h" #include "BLI_memarena.h" #include "BLI_mempool.h" -#include "BLI_mmap.h" #include "BLI_threads.h" #include "PIL_time.h" @@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd) */ if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) { bhead4.code = DATA; - readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL); + readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4)); if (readsize == sizeof(bhead4) || bhead4.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd) } else { bhead8.code = DATA; - readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL); + readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8)); if (readsize == sizeof(bhead8) || bhead8.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd) /* pass */ } #ifdef USE_BHEAD_READ_ON_DEMAND - else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { + else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { /* Delay reading bhead content. */ new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead"); if (new_bhead) { new_bhead->next = new_bhead->prev = NULL; - new_bhead->file_offset = fd->file_offset; + new_bhead->file_offset = fd->file->offset; new_bhead->has_data = false; new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR); + off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR); if (seek_new == -1) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } - BLI_assert(fd->file_offset == seek_new); + BLI_assert(fd->file->offset == seek_new); } else { fd->is_eof = true; @@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd) new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - readsize = fd->read( - fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical); + readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len); - if (readsize != (ssize_t)bhead.len) { + if (readsize != bhead.len) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } + + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } else { fd->is_eof = true; @@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf) bool success = true; BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock); BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0); - off64_t offset_backup = fd->file_offset; - if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) { + off64_t offset_backup = fd->file->offset; + if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) { success = false; } else { - if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) != - (ssize_t)new_bhead->bhead.len) { + if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) { success = false; } + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } - if (fd->seek(fd, offset_backup, SEEK_SET) == -1) { + if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) { success = false; } return success; @@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd) ssize_t readsize; /* read in the header data */ - readsize = fd->read(fd, header, sizeof(header), NULL); + readsize = fd->file->read(fd->file, header, sizeof(header)); if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') && ELEM(header[8], 'v', 'V') && @@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd) /** \} */ -/* -------------------------------------------------------------------- */ -/** \name File Data API - * \{ */ - -/* Regular file reading. */ - -static ssize_t fd_read_data_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - ssize_t readsize = read(filedata->filedes, buffer, size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence) -{ - filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence); - return filedata->file_offset; -} - -/* GZip file reading. */ - -static ssize_t fd_read_gzip_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - BLI_assert(size <= INT_MAX); - - ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -/* Memory reading. */ - -static ssize_t fd_read_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset); - - memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize); - filedata->file_offset += readsize; - - return readsize; -} - -/* Memory-mapped file reading. - * By using mmap(), we can map a file so that it can be treated like normal memory, - * meaning that we can just read from it with memcpy() etc. - * This avoids system call overhead and can significantly speed up file loading. - */ - -static ssize_t fd_read_from_mmap(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset)); - - if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) { - return 0; - } - - filedata->file_offset += readsize; - - return readsize; -} - -static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence) -{ - off64_t new_pos; - if (whence == SEEK_CUR) { - new_pos = filedata->file_offset + offset; - } - else if (whence == SEEK_SET) { - new_pos = offset; - } - else if (whence == SEEK_END) { - new_pos = filedata->buffersize + offset; - } - else { - return -1; - } - - if (new_pos < 0 || new_pos > filedata->buffersize) { - return -1; - } - - filedata->file_offset = new_pos; - return filedata->file_offset; -} - -/* MemFile reading. */ - -static ssize_t fd_read_from_memfile(FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunck_identical) -{ - static size_t seek = SIZE_MAX; /* the current position */ - static size_t offset = 0; /* size of previous chunks */ - static MemFileChunk *chunk = NULL; - size_t chunkoffset, readsize, totread; - - if (r_is_memchunck_identical != NULL) { - *r_is_memchunck_identical = true; - } - - if (size == 0) { - return 0; - } - - if (seek != (size_t)filedata->file_offset) { - chunk = filedata->memfile->chunks.first; - seek = 0; - - while (chunk) { - if (seek + chunk->size > (size_t)filedata->file_offset) { - break; - } - seek += chunk->size; - chunk = chunk->next; - } - offset = seek; - seek = (size_t)filedata->file_offset; - } - - if (chunk) { - totread = 0; - - do { - /* first check if it's on the end if current chunk */ - if (seek - offset == chunk->size) { - offset += chunk->size; - chunk = chunk->next; - } - - /* debug, should never happen */ - if (chunk == NULL) { - CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk"); - return 0; - } - - chunkoffset = seek - offset; - readsize = size - totread; - - /* data can be spread over multiple chunks, so clamp size - * to within this chunk, and then it will read further in - * the next chunk */ - if (chunkoffset + readsize > chunk->size) { - readsize = chunk->size - chunkoffset; - } - - memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); - totread += readsize; - filedata->file_offset += readsize; - seek += readsize; - if (r_is_memchunck_identical != NULL) { - /* `is_identical` of current chunk represents whether it changed compared to previous undo - * step. this is fine in redo case, but not in undo case, where we need an extra flag - * defined when saving the next (future) step after the one we want to restore, as we are - * supposed to 'come from' that future undo step, and not the one before current one. */ - *r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ? - chunk->is_identical : - chunk->is_identical_future; - } - } while (totread < size); - - return (ssize_t)totread; - } - - return 0; -} - static FileData *filedata_new(BlendFileReadReport *reports) { BLI_assert(reports != NULL); FileData *fd = MEM_callocN(sizeof(FileData), "FileData"); - fd->filedes = -1; - fd->gzfiledes = NULL; - fd->memsdna = DNA_sdna_current_get(); fd->datamap = oldnewmap_new(); @@ -1387,78 +1191,60 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports) static FileData *blo_filedata_from_file_descriptor(const char *filepath, BlendFileReadReport *reports, - int file) + int filedes) { - FileDataReadFn *read_fn = NULL; - FileDataSeekFn *seek_fn = NULL; /* Optional. */ - size_t buffersize = 0; - BLI_mmap_file *mmap_file = NULL; - - gzFile gzfile = (gzFile)Z_NULL; - char header[7]; + FileReader *rawfile = BLI_filereader_new_file(filedes); + FileReader *file = NULL; - /* Regular file. */ errno = 0; - if (read(file, header, sizeof(header)) != sizeof(header)) { + /* If opening the file failed or we can't read the header, give up. */ + if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) { BKE_reportf(reports->reports, RPT_WARNING, "Unable to read '%s': %s", filepath, errno ? strerror(errno) : TIP_("insufficient content")); + if (rawfile) { + rawfile->close(rawfile); + } + else { + close(filedes); + } return NULL; } - /* Regular file. */ - if (memcmp(header, "BLENDER", sizeof(header)) == 0) { - read_fn = fd_read_data_from_file; - seek_fn = fd_seek_data_from_file; + /* Rewind the file after reading the header. */ + rawfile->seek(rawfile, 0, SEEK_SET); - mmap_file = BLI_mmap_open(file); - if (mmap_file != NULL) { - read_fn = fd_read_from_mmap; - seek_fn = fd_seek_from_mmap; - buffersize = BLI_lseek(file, 0, SEEK_END); + /* Check if we have a regular file. */ + if (memcmp(header, "BLENDER", sizeof(header)) == 0) { + /* Try opening the file with memory-mapped IO. */ + file = BLI_filereader_new_mmap(filedes); + if (file == NULL) { + /* mmap failed, so just keep using rawfile. */ + file = rawfile; + rawfile = NULL; } } - - BLI_lseek(file, 0, SEEK_SET); - - /* Gzip file. */ - errno = 0; - if ((read_fn == NULL) && - /* Check header magic. */ - (header[0] == 0x1f && header[1] == 0x8b)) { - gzfile = BLI_gzopen(filepath, "rb"); - if (gzfile == (gzFile)Z_NULL) { - BKE_reportf(reports->reports, - RPT_WARNING, - "Unable to open '%s': %s", - filepath, - errno ? strerror(errno) : TIP_("unknown error reading file")); - return NULL; + else if (BLI_file_magic_is_gzip(header)) { + file = BLI_filereader_new_gzip(rawfile); + if (file != NULL) { + rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */ } - - /* 'seek_fn' is too slow for gzip, don't set it. */ - read_fn = fd_read_gzip_from_file; - /* Caller must close. */ - file = -1; } - if (read_fn == NULL) { + /* Clean up `rawfile` if it wasn't taken over. */ + if (rawfile != NULL) { + rawfile->close(rawfile); + } + if (file == NULL) { BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath); return NULL; } FileData *fd = filedata_new(reports); - - fd->filedes = file; - fd->gzfiledes = gzfile; - - fd->read = read_fn; - fd->seek = seek_fn; - fd->mmap_file = mmap_file; - fd->buffersize = buffersize; + fd->file = file; return fd; } @@ -1475,11 +1261,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead errno ? strerror(errno) : TIP_("unknown error reading file")); return NULL; } - FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file); - if ((fd == NULL) || (fd->filedes == -1)) { - close(file); - } - return fd; + return blo_filedata_from_file_descriptor(filepath, reports, file); } /* cannot be called with relative paths anymore! */ @@ -1513,50 +1295,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath) return NULL; } -static ssize_t fd_read_gzip_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - int err; - - filedata->strm.next_out = (Bytef *)buffer; - filedata->strm.avail_out = (uint)size; - - /* Inflate another chunk. */ - err = inflate(&filedata->strm, Z_SYNC_FLUSH); - - if (err == Z_STREAM_END) { - return 0; - } - if (err != Z_OK) { - CLOG_ERROR(&LOG, "ZLib error (code %d)", err); - return 0; - } - - filedata->file_offset += size; - - return (ssize_t)size; -} - -static int fd_read_gzip_from_memory_init(FileData *fd) -{ - - fd->strm.next_in = (Bytef *)fd->buffer; - fd->strm.avail_in = fd->buffersize; - fd->strm.total_out = 0; - fd->strm.zalloc = Z_NULL; - fd->strm.zfree = Z_NULL; - - if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) { - return 0; - } - - fd->read = fd_read_gzip_from_memory; - - return 1; -} - FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports) { if (!mem || memsize < SIZEOFBLENDERHEADER) { @@ -1565,24 +1303,21 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe return NULL; } - FileData *fd = filedata_new(reports); - const char *cp = mem; - - fd->buffer = mem; - fd->buffersize = memsize; + FileReader *mem_file = BLI_filereader_new_memory(mem, memsize); + FileReader *file = mem_file; - /* test if gzip */ - if (cp[0] == 0x1f && cp[1] == 0x8b) { - if (0 == fd_read_gzip_from_memory_init(fd)) { - blo_filedata_free(fd); - return NULL; - } + if (BLI_file_magic_is_gzip(mem)) { + file = BLI_filereader_new_gzip(mem_file); } - else { - fd->read = fd_read_from_memory; + + if (file == NULL) { + /* Compression initialization failed. */ + mem_file->close(mem_file); + return NULL; } - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + FileData *fd = filedata_new(reports); + fd->file = file; return blo_decode_and_check(fd, reports->reports); } @@ -1597,11 +1332,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, } FileData *fd = filedata_new(reports); - fd->memfile = memfile; + fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction); fd->undo_direction = params->undo_direction; - - fd->read = fd_read_from_memfile; - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + fd->flags |= FD_FLAGS_IS_MEMFILE; return blo_decode_and_check(fd, reports->reports); } @@ -1609,30 +1342,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, void blo_filedata_free(FileData *fd) { if (fd) { - if (fd->filedes != -1) { - close(fd->filedes); - } - - if (fd->gzfiledes != NULL) { - gzclose(fd->gzfiledes); - } - - if (fd->strm.next_in) { - int err = inflateEnd(&fd->strm); - if (err != Z_OK) { - CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err); - } - } - - if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) { - MEM_freeN((void *)fd->buffer); - fd->buffer = NULL; - } - - if (fd->mmap_file) { - BLI_mmap_free(fd->mmap_file); - fd->mmap_file = NULL; - } + fd->file->close(fd->file); /* Free all BHeadN data blocks */ #ifndef NDEBUG @@ -1640,7 +1350,7 @@ void blo_filedata_free(FileData *fd) #else /* Sanity check we're not keeping memory we don't need. */ LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) { - if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { + if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { BLI_assert(new_bhead->has_data == 0); } MEM_freeN(new_bhead); @@ -2096,7 +1806,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id), void blo_cache_storage_init(FileData *fd, Main *bmain) { - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { BLI_assert(fd->cache_storage == NULL); fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__); fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__); @@ -2261,7 +1971,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname) * undo since DNA must match. */ static const void *peek_struct_undo(FileData *fd, BHead *bhead) { - BLI_assert(fd->memfile != NULL); + BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE); UNUSED_VARS_NDEBUG(fd); return (bhead->len) ? (const void *)(bhead + 1) : NULL; } @@ -3679,7 +3389,7 @@ static BHead *read_libblock(FileData *fd, * When datablocks are changed but still exist, we restore them at the old * address and inherit recalc flags for the dependency graph. */ ID *id_old = NULL; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) { if (r_id) { *r_id = id_old; @@ -3980,13 +3690,14 @@ static void lib_link_all(FileData *fd, Main *bmain) continue; } - if (fd->memfile != NULL && GS(id->name) == ID_WM) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) { /* No load UI for undo memfiles. * Only WM currently, SCR needs it still (see below), and so does WS? */ continue; } - if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo && + (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { /* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across * current undo step, and old IDs re-use their old memory address, we do not need to liblink * it at all. */ @@ -4165,7 +3876,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) BlendFileData *bfd; ListBase mainlist = {NULL, NULL}; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step"); } @@ -4256,7 +3967,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } /* do before read_libraries, but skip undo case */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) { do_versions(fd, NULL, bfd->main); } @@ -4278,7 +3989,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries; /* Skip in undo case. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Note that we can't recompute user-counts at this point in undo case, we play too much with * IDs from different memory realms, and Main database is not in a fully valid state yet. */ @@ -4311,7 +4022,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) /* Now that all our data-blocks are loaded, * we can re-generate overrides from their references. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Do not apply in undo case! */ fd->reports->duration.lib_overrides = PIL_check_seconds_timer(); @@ -4391,7 +4102,7 @@ static void sort_bhead_old_map(FileData *fd) static BHead *find_previous_lib(FileData *fd, BHead *bhead) { /* Skip library data-blocks in undo, see comment in read_libblock. */ - if (fd->memfile) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { return NULL; } @@ -5850,7 +5561,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p) bool BLO_read_data_is_undo(BlendDataReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr) @@ -5870,7 +5581,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader) bool BLO_read_lib_is_undo(BlendLibReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } Main *BLO_read_lib_get_main(BlendLibReader *reader) diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h index b04043f9641..beeed8e45ae 100644 --- a/source/blender/blenloader/intern/readfile.h +++ b/source/blender/blenloader/intern/readfile.h @@ -28,10 +28,10 @@ # include "BLI_winstuff.h" #endif +#include "BLI_filereader.h" #include "DNA_sdna_types.h" #include "DNA_space_types.h" #include "DNA_windowmanager_types.h" /* for ReportType */ -#include "zlib.h" struct BLI_mmap_file; struct BLOCacheStorage; @@ -50,7 +50,7 @@ enum eFileDataFlag { FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1, FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2, FD_FLAGS_FILE_OK = 1 << 3, - FD_FLAGS_NOT_MY_BUFFER = 1 << 4, + FD_FLAGS_IS_MEMFILE = 1 << 4, /* XXX Unused in practice (checked once but never set). */ FD_FLAGS_NOT_MY_LIBMAP = 1 << 5, }; @@ -60,44 +60,18 @@ enum eFileDataFlag { # pragma GCC poison off_t #endif -#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__) -typedef int64_t off64_t; -#endif - -typedef ssize_t(FileDataReadFn)(struct FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunk_identical); -typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence); - typedef struct FileData { /** Linked list of BHeadN's. */ ListBase bhead_list; enum eFileDataFlag flags; bool is_eof; - size_t buffersize; - off64_t file_offset; - FileDataReadFn *read; - FileDataSeekFn *seek; + FileReader *file; - /** Regular file reading. */ - int filedes; - - /** Variables needed for reading from memory / stream / memory-mapped files. */ - const char *buffer; - struct BLI_mmap_file *mmap_file; - /** Variables needed for reading from memfile (undo). */ - struct MemFile *memfile; /** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use * to detect unchanged data from memfile. */ int undo_direction; /* eUndoStepDir */ - /** Variables needed for reading from file. */ - gzFile gzfiledes; - /** Gzip stream for memory decompression. */ - z_stream strm; - /** Now only in use for library appending. */ char relabase[FILE_MAX]; diff --git a/source/blender/blenloader/intern/undofile.c b/source/blender/blenloader/intern/undofile.c index 2eeeac2e8d7..62072cf7df5 100644 --- a/source/blender/blenloader/intern/undofile.c +++ b/source/blender/blenloader/intern/undofile.c @@ -48,6 +48,7 @@ #include "BKE_lib_id.h" #include "BKE_main.h" +#include "BKE_undo_system.h" /* keep last */ #include "BLI_strict_flags.h" @@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename) } return true; } + +static ssize_t undo_read(FileReader *reader, void *buffer, size_t size) +{ + UndoReader *undo = (UndoReader *)reader; + + static size_t seek = SIZE_MAX; /* The current position. */ + static size_t offset = 0; /* Size of previous chunks. */ + static MemFileChunk *chunk = NULL; + size_t chunkoffset, readsize, totread; + + undo->memchunk_identical = true; + + if (size == 0) { + return 0; + } + + if (seek != (size_t)undo->reader.offset) { + chunk = undo->memfile->chunks.first; + seek = 0; + + while (chunk) { + if (seek + chunk->size > (size_t)undo->reader.offset) { + break; + } + seek += chunk->size; + chunk = chunk->next; + } + offset = seek; + seek = (size_t)undo->reader.offset; + } + + if (chunk) { + totread = 0; + + do { + /* First check if it's on the end if current chunk. */ + if (seek - offset == chunk->size) { + offset += chunk->size; + chunk = chunk->next; + } + + /* Debug, should never happen. */ + if (chunk == NULL) { + printf("illegal read, chunk zero\n"); + return 0; + } + + chunkoffset = seek - offset; + readsize = size - totread; + + /* Data can be spread over multiple chunks, so clamp size + * to within this chunk, and then it will read further in + * the next chunk. */ + if (chunkoffset + readsize > chunk->size) { + readsize = chunk->size - chunkoffset; + } + + memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); + totread += readsize; + undo->reader.offset += (off64_t)readsize; + seek += readsize; + + /* `is_identical` of current chunk represents whether it changed compared to previous undo + * step. this is fine in redo case, but not in undo case, where we need an extra flag + * defined when saving the next (future) step after the one we want to restore, as we are + * supposed to 'come from' that future undo step, and not the one before current one. */ + undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical : + chunk->is_identical_future; + } while (totread < size); + + return (ssize_t)totread; + } + + return 0; +} + +static void undo_close(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction) +{ + UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__); + + undo->memfile = memfile; + undo->undo_direction = undo_direction; + + undo->reader.read = undo_read; + undo->reader.seek = NULL; + undo->reader.close = undo_close; + + return (FileReader *)undo; +} diff --git a/source/blender/blenloader/intern/versioning_250.c b/source/blender/blenloader/intern/versioning_250.c index e56c1995363..436645c2241 100644 --- a/source/blender/blenloader/intern/versioning_250.c +++ b/source/blender/blenloader/intern/versioning_250.c @@ -23,8 +23,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include /* for open close read */ -# include /* odd include order-issue */ +# include /* for open close read */ #endif /* allow readfile to use deprecated functionality */ diff --git a/source/blender/blenloader/intern/versioning_legacy.c b/source/blender/blenloader/intern/versioning_legacy.c index 81371e1c1ed..6ba27b6ee9e 100644 --- a/source/blender/blenloader/intern/versioning_legacy.c +++ b/source/blender/blenloader/intern/versioning_legacy.c @@ -28,8 +28,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include /* for open close read */ -# include /* odd include order-issue */ +# include /* for open close read */ #endif /* allow readfile to use deprecated functionality */ diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c index 99246603e9a..4de8e35f402 100644 --- a/source/blender/blenloader/intern/writefile.c +++ b/source/blender/blenloader/intern/writefile.c @@ -78,12 +78,12 @@ #include #include #include +#include #ifdef WIN32 # include "BLI_winstuff.h" # include "winsock2.h" # include -# include /* odd include order-issue */ #else # include /* FreeBSD, for write() and close(). */ #endif diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c index b53ad0ee927..c3789abf2dd 100644 --- a/source/blender/windowmanager/intern/wm_files.c +++ b/source/blender/windowmanager/intern/wm_files.c @@ -28,11 +28,10 @@ * winsock stuff. */ #include +#include /* for open flags (O_BINARY, O_RDONLY). */ #include #include -#include "zlib.h" /* wm_read_exotic() */ - #ifdef WIN32 /* Need to include windows.h so _WIN32_IE is defined. */ # include @@ -51,6 +50,7 @@ #include "BLI_blenlib.h" #include "BLI_fileops_types.h" +#include "BLI_filereader.h" #include "BLI_linklist.h" #include "BLI_math.h" #include "BLI_system.h" @@ -481,53 +481,59 @@ static void wm_init_userdef(Main *bmain) /* intended to check for non-blender formats but for now it only reads blends */ static int wm_read_exotic(const char *name) { - int len; - gzFile gzfile; + /* make sure we're not trying to read a directory.... */ + + int namelen = strlen(name); + if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) { + return BKE_READ_EXOTIC_FAIL_PATH; + } + + /* open the file. */ + const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0); + if (filedes == -1) { + return BKE_READ_EXOTIC_FAIL_OPEN; + } + + FileReader *rawfile = BLI_filereader_new_file(filedes); + if (rawfile == NULL) { + return BKE_READ_EXOTIC_FAIL_OPEN; + } + + /* read the header (7 bytes are enough to identify all known types). */ char header[7]; - int retval; + if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) { + rawfile->close(rawfile); + return BKE_READ_EXOTIC_FAIL_FORMAT; + } + rawfile->seek(rawfile, 0, SEEK_SET); - /* make sure we're not trying to read a directory.... */ + /* check for uncompressed .blend */ + if (STREQLEN(header, "BLENDER", 7)) { + rawfile->close(rawfile); + return BKE_READ_EXOTIC_OK_BLEND; + } - len = strlen(name); - if (len > 0 && ELEM(name[len - 1], '/', '\\')) { - retval = BKE_READ_EXOTIC_FAIL_PATH; + /* check for compressed .blend */ + FileReader *compressed_file = NULL; + if (BLI_file_magic_is_gzip(header)) { + compressed_file = BLI_filereader_new_gzip(rawfile); } - else { - gzfile = BLI_gzopen(name, "rb"); - if (gzfile == NULL) { - retval = BKE_READ_EXOTIC_FAIL_OPEN; - } - else { - len = gzread(gzfile, header, sizeof(header)); - gzclose(gzfile); - if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) { - retval = BKE_READ_EXOTIC_OK_BLEND; - } - else { - /* We may want to support loading other file formats - * from their header bytes or file extension. - * This used to be supported in the code below and may be added - * back at some point. */ -#if 0 - WM_cursor_wait(true); - if (is_foo_format(name)) { - read_foo(name); - retval = BKE_READ_EXOTIC_OK_OTHER; - } - else -#endif - { - retval = BKE_READ_EXOTIC_FAIL_FORMAT; - } -#if 0 - WM_cursor_wait(false); -#endif - } + /* If a compression signature matches, try decompressing the start and check if it's a .blend */ + if (compressed_file != NULL) { + size_t len = compressed_file->read(compressed_file, header, sizeof(header)); + compressed_file->close(compressed_file); + if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) { + return BKE_READ_EXOTIC_OK_BLEND; } } + else { + rawfile->close(rawfile); + } + + /* Add check for future file formats here. */ - return retval; + return BKE_READ_EXOTIC_FAIL_FORMAT; } /** \} */ -- cgit v1.2.3