diff options
author | Lukas Stockner <lukas.stockner@freenet.de> | 2021-08-20 00:57:00 +0300 |
---|---|---|
committer | Lukas Stockner <lukas.stockner@freenet.de> | 2021-08-21 22:38:57 +0300 |
commit | 2b170f16d6ded9b3bcb428121b27274ae8637555 (patch) | |
tree | 3764bd1345eda9843658b8843acd5984f4077b90 /source/blender/blenlib/intern | |
parent | 34a05f39be2b79dd1c508c374a47cee6792174f9 (diff) |
Refactor low-level blendfile reading into separate files
Instead of handling mmap, compression etc. all directly in readfile.c, refactor
the code to use a generic FileReader.
This makes it easier to add new compression methods or similar, and allows to
reuse the logic in other places (e.g. thumbnail reading).
Reviewed By: campbellbarton, brecht, mont29
Differential Revision: https://developer.blender.org/D5799
Diffstat (limited to 'source/blender/blenlib/intern')
-rw-r--r-- | source/blender/blenlib/intern/fileops.c | 7 | ||||
-rw-r--r-- | source/blender/blenlib/intern/filereader_file.c | 80 | ||||
-rw-r--r-- | source/blender/blenlib/intern/filereader_gzip.c | 108 | ||||
-rw-r--r-- | source/blender/blenlib/intern/filereader_memory.c | 145 |
4 files changed, 340 insertions, 0 deletions
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c index ac034d2b5cd..6fc2222241b 100644 --- a/source/blender/blenlib/intern/fileops.c +++ b/source/blender/blenlib/intern/fileops.c @@ -255,6 +255,13 @@ size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t g #undef CHUNK +bool BLI_file_magic_is_gzip(const char header[4]) +{ + /* GZIP itself starts with the magic bytes 0x1f 0x8b. + * The third byte indicates the compression method, which is 0x08 for DEFLATE. */ + return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08; +} + /** * Returns true if the file with the specified name can be written. * This implementation uses access(2), which makes the check according diff --git a/source/blender/blenlib/intern/filereader_file.c b/source/blender/blenlib/intern/filereader_file.c new file mode 100644 index 00000000000..3a833871e27 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_file.c @@ -0,0 +1,80 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#ifndef WIN32 +# include <unistd.h> /* for read close */ +#else +# include "BLI_winstuff.h" +# include "winsock2.h" +# include <io.h> /* for open close read */ +#endif + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + int filedes; +} RawFileReader; + +static ssize_t file_read(FileReader *reader, void *buffer, size_t size) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + ssize_t readsize = read(rawfile->filedes, buffer, size); + + if (readsize >= 0) { + rawfile->reader.offset += readsize; + } + + return readsize; +} + +static off64_t file_seek(FileReader *reader, off64_t offset, int whence) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence); + return rawfile->reader.offset; +} + +static void file_close(FileReader *reader) +{ + RawFileReader *rawfile = (RawFileReader *)reader; + close(rawfile->filedes); + MEM_freeN(rawfile); +} + +FileReader *BLI_filereader_new_file(int filedes) +{ + RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__); + + rawfile->filedes = filedes; + + rawfile->reader.read = file_read; + rawfile->reader.seek = file_seek; + rawfile->reader.close = file_close; + + return (FileReader *)rawfile; +} diff --git a/source/blender/blenlib/intern/filereader_gzip.c b/source/blender/blenlib/intern/filereader_gzip.c new file mode 100644 index 00000000000..72eb153a8b9 --- /dev/null +++ b/source/blender/blenlib/intern/filereader_gzip.c @@ -0,0 +1,108 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include <zlib.h> + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" + +#include "MEM_guardedalloc.h" + +typedef struct { + FileReader reader; + + FileReader *base; + + z_stream strm; + + void *in_buf; + size_t in_size; +} GzipReader; + +static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size) +{ + GzipReader *gzip = (GzipReader *)reader; + + gzip->strm.avail_out = size; + gzip->strm.next_out = buffer; + + while (gzip->strm.avail_out > 0) { + if (gzip->strm.avail_in == 0) { + /* Ran out of buffered input data, read some more. */ + size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size); + + if (readsize > 0) { + /* We got some data, so mark the buffer as refilled. */ + gzip->strm.avail_in = readsize; + gzip->strm.next_in = gzip->in_buf; + } + else { + /* The underlying file is EOF, so return as much as we can. */ + break; + } + } + + int ret = inflate(&gzip->strm, Z_NO_FLUSH); + + if (ret != Z_OK && ret != Z_BUF_ERROR) { + break; + } + } + + ssize_t read_len = size - gzip->strm.avail_out; + gzip->reader.offset += read_len; + return read_len; +} + +static void gzip_close(FileReader *reader) +{ + GzipReader *gzip = (GzipReader *)reader; + + if (inflateEnd(&gzip->strm) != Z_OK) { + printf("close gzip stream error\n"); + } + MEM_freeN((void *)gzip->in_buf); + + gzip->base->close(gzip->base); + MEM_freeN(gzip); +} + +FileReader *BLI_filereader_new_gzip(FileReader *base) +{ + GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__); + gzip->base = base; + + if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) { + MEM_freeN(gzip); + return NULL; + } + + gzip->in_size = 256 * 2014; + gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf"); + + gzip->reader.read = gzip_read; + gzip->reader.seek = NULL; + gzip->reader.close = gzip_close; + + return (FileReader *)gzip; +} diff --git a/source/blender/blenlib/intern/filereader_memory.c b/source/blender/blenlib/intern/filereader_memory.c new file mode 100644 index 00000000000..150fed7d1cc --- /dev/null +++ b/source/blender/blenlib/intern/filereader_memory.c @@ -0,0 +1,145 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2004-2021 Blender Foundation + * All rights reserved. + */ + +/** \file + * \ingroup bli + */ + +#include <string.h> + +#include "BLI_blenlib.h" +#include "BLI_filereader.h" +#include "BLI_mmap.h" + +#include "MEM_guardedalloc.h" + +/* This file implements both memory-backed and memory-mapped-file-backed reading. */ +typedef struct { + FileReader reader; + + const char *data; + BLI_mmap_file *mmap; + size_t length; +} MemoryReader; + +static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + memcpy(buffer, mem->data + mem->reader.offset, readsize); + mem->reader.offset += readsize; + + return readsize; +} + +static off64_t memory_seek(FileReader *reader, off64_t offset, int whence) +{ + MemoryReader *mem = (MemoryReader *)reader; + + off64_t new_pos; + if (whence == SEEK_CUR) { + new_pos = mem->reader.offset + offset; + } + else if (whence == SEEK_SET) { + new_pos = offset; + } + else if (whence == SEEK_END) { + new_pos = mem->length + offset; + } + else { + return -1; + } + + if (new_pos < 0 || new_pos > mem->length) { + return -1; + } + + mem->reader.offset = new_pos; + return mem->reader.offset; +} + +static void memory_close_raw(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLI_filereader_new_memory(const void *data, size_t len) +{ + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->data = (const char *)data; + mem->length = len; + + mem->reader.read = memory_read_raw; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_raw; + + return (FileReader *)mem; +} + +/* Memory-mapped file reading. + * By using `mmap()`, we can map a file so that it can be treated like normal memory, + * meaning that we can just read from it with `memcpy()` etc. + * This avoids system call overhead and can significantly speed up file loading. + */ + +static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size) +{ + MemoryReader *mem = (MemoryReader *)reader; + + /* Don't read more bytes than there are available in the buffer. */ + size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset)); + + if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) { + return 0; + } + + mem->reader.offset += readsize; + + return readsize; +} + +static void memory_close_mmap(FileReader *reader) +{ + MemoryReader *mem = (MemoryReader *)reader; + BLI_mmap_free(mem->mmap); + MEM_freeN(mem); +} + +FileReader *BLI_filereader_new_mmap(int filedes) +{ + BLI_mmap_file *mmap = BLI_mmap_open(filedes); + if (mmap == NULL) { + return NULL; + } + + MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__); + + mem->mmap = mmap; + mem->length = BLI_lseek(filedes, 0, SEEK_END); + + mem->reader.read = memory_read_mmap; + mem->reader.seek = memory_seek; + mem->reader.close = memory_close_mmap; + + return (FileReader *)mem; +} |