Sanitize type 'size' parameters in our read/write file code

This patch tries to sanitize the types of our size parameters across our read
and write code, which is currently fairly inconsistent (using `int`, `uint`,
`size_t`...), by using `size_t` everywhere. Since in Blender file themselves
we can only store chunk of size `MAX_INT`, added some asserts to ensure that
as well.

See {T79561} for details.

Differential Revision: https://developer.blender.org/D8672

4 files changed, 84 insertions, 71 deletions

diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index bc04720d5e4..6e0fc1752a9 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -809,7 +809,7 @@ static void bh8_from_bh4(BHead *bhead, BHead4 *bhead4)
 static BHeadN *get_bhead(FileData *fd)
 {
   BHeadN *new_bhead = NULL;
-  int readsize;
+  ssize_t readsize;
 
   if (fd) {
     if (!fd->is_eof) {
@@ -908,7 +908,7 @@ static BHeadN *get_bhead(FileData *fd)
       }
 #endif
       else {
-        new_bhead = MEM_mallocN(sizeof(BHeadN) + bhead.len, "new_bhead");
+        new_bhead = MEM_mallocN(sizeof(BHeadN) + (size_t)bhead.len, "new_bhead");
         if (new_bhead) {
           new_bhead->next = new_bhead->prev = NULL;
 #ifdef USE_BHEAD_READ_ON_DEMAND
@@ -918,9 +918,10 @@ static BHeadN *get_bhead(FileData *fd)
           new_bhead->is_memchunk_identical = false;
           new_bhead->bhead = bhead;
 
-          readsize = fd->read(fd, new_bhead + 1, bhead.len, &new_bhead->is_memchunk_identical);
+          readsize = fd->read(
+              fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
 
-          if (readsize != bhead.len) {
+          if (readsize != (ssize_t)bhead.len) {
             fd->is_eof = true;
             MEM_freeN(new_bhead);
             new_bhead = NULL;
@@ -1008,8 +1009,8 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
     success = false;
   }
   else {
-    if (fd->read(fd, buf, new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
-        new_bhead->bhead.len) {
+    if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
+        (ssize_t)new_bhead->bhead.len) {
       success = false;
     }
   }
@@ -1044,7 +1045,7 @@ const char *blo_bhead_id_name(const FileData *fd, const BHead *bhead)
 static void decode_blender_header(FileData *fd)
 {
   char header[SIZEOFBLENDERHEADER], num[4];
-  int readsize;
+  ssize_t readsize;
 
   /* read in the header data */
   readsize = fd->read(fd, header, sizeof(header), NULL);
@@ -1178,12 +1179,12 @@ static int *read_file_thumbnail(FileData *fd)
 
 /* Regular file reading. */
 
-static int fd_read_data_from_file(FileData *filedata,
-                                  void *buffer,
-                                  uint size,
-                                  bool *UNUSED(r_is_memchunck_identical))
+static ssize_t fd_read_data_from_file(FileData *filedata,
+                                      void *buffer,
+                                      size_t size,
+                                      bool *UNUSED(r_is_memchunck_identical))
 {
-  int readsize = read(filedata->filedes, buffer, size);
+  ssize_t readsize = read(filedata->filedes, buffer, size);
 
   if (readsize < 0) {
     readsize = EOF;
@@ -1203,12 +1204,14 @@ static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int wh
 
 /* GZip file reading. */
 
-static int fd_read_gzip_from_file(FileData *filedata,
-                                  void *buffer,
-                                  uint size,
-                                  bool *UNUSED(r_is_memchunck_identical))
+static ssize_t fd_read_gzip_from_file(FileData *filedata,
+                                      void *buffer,
+                                      size_t size,
+                                      bool *UNUSED(r_is_memchunck_identical))
 {
-  int readsize = gzread(filedata->gzfiledes, buffer, size);
+  BLI_assert(size <= INT_MAX);
+
+  ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
 
   if (readsize < 0) {
     readsize = EOF;
@@ -1222,15 +1225,15 @@ static int fd_read_gzip_from_file(FileData *filedata,
 
 /* Memory reading. */
 
-static int fd_read_from_memory(FileData *filedata,
-                               void *buffer,
-                               uint size,
-                               bool *UNUSED(r_is_memchunck_identical))
+static ssize_t fd_read_from_memory(FileData *filedata,
+                                   void *buffer,
+                                   size_t size,
+                                   bool *UNUSED(r_is_memchunck_identical))
 {
   /* don't read more bytes then there are available in the buffer */
-  int readsize = (int)MIN2(size, (uint)(filedata->buffersize - filedata->file_offset));
+  ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
 
-  memcpy(buffer, filedata->buffer + filedata->file_offset, readsize);
+  memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
   filedata->file_offset += readsize;
 
   return readsize;
@@ -1238,10 +1241,10 @@ static int fd_read_from_memory(FileData *filedata,
 
 /* MemFile reading. */
 
-static int fd_read_from_memfile(FileData *filedata,
-                                void *buffer,
-                                uint size,
-                                bool *r_is_memchunck_identical)
+static ssize_t fd_read_from_memfile(FileData *filedata,
+                                    void *buffer,
+                                    size_t size,
+                                    bool *r_is_memchunck_identical)
 {
   static size_t seek = SIZE_MAX; /* the current position */
   static size_t offset = 0;      /* size of previous chunks */
@@ -1268,7 +1271,7 @@ static int fd_read_from_memfile(FileData *filedata,
       chunk = chunk->next;
     }
     offset = seek;
-    seek = filedata->file_offset;
+    seek = (size_t)filedata->file_offset;
   }
 
   if (chunk) {
@@ -1312,7 +1315,7 @@ static int fd_read_from_memfile(FileData *filedata,
       }
     } while (totread < size);
 
-    return totread;
+    return (ssize_t)totread;
   }
 
   return 0;
@@ -1474,15 +1477,15 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
   return NULL;
 }
 
-static int fd_read_gzip_from_memory(FileData *filedata,
-                                    void *buffer,
-                                    uint size,
-                                    bool *UNUSED(r_is_memchunck_identical))
+static ssize_t fd_read_gzip_from_memory(FileData *filedata,
+                                        void *buffer,
+                                        size_t size,
+                                        bool *UNUSED(r_is_memchunck_identical))
 {
   int err;
 
   filedata->strm.next_out = (Bytef *)buffer;
-  filedata->strm.avail_out = size;
+  filedata->strm.avail_out = (uint)size;
 
   // Inflate another chunk.
   err = inflate(&filedata->strm, Z_SYNC_FLUSH);
@@ -1497,7 +1500,7 @@ static int fd_read_gzip_from_memory(FileData *filedata,
 
   filedata->file_offset += size;
 
-  return size;
+  return (ssize_t)size;
 }
 
 static int fd_read_gzip_from_memory_init(FileData *fd)
diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h
index e63b0c1909b..c479e3e589b 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -61,10 +61,10 @@ enum eFileDataFlag {
 typedef int64_t off64_t;
 #endif
 
-typedef int(FileDataReadFn)(struct FileData *filedata,
-                            void *buffer,
-                            unsigned int size,
-                            bool *r_is_memchunk_identical);
+typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
+                                void *buffer,
+                                size_t size,
+                                bool *r_is_memchunk_identical);
 typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
 
 typedef struct FileData {
@@ -72,8 +72,8 @@ typedef struct FileData {
   ListBase bhead_list;
   enum eFileDataFlag flags;
   bool is_eof;
-  int buffersize;
-  int64_t file_offset;
+  size_t buffersize;
+  off64_t file_offset;
 
   FileDataReadFn *read;
   FileDataSeekFn *seek;
diff --git a/source/blender/blenloader/intern/undofile.c b/source/blender/blenloader/intern/undofile.c
index 8ee539014f0..4da73138c38 100644
--- a/source/blender/blenloader/intern/undofile.c
+++ b/source/blender/blenloader/intern/undofile.c
@@ -154,7 +154,7 @@ void BLO_memfile_write_finalize(MemFileWriteData *mem_data)
   }
 }
 
-void BLO_memfile_chunk_add(MemFileWriteData *mem_data, const char *buf, uint size)
+void BLO_memfile_chunk_add(MemFileWriteData *mem_data, const char *buf, size_t size)
 {
   MemFile *memfile = mem_data->written_memfile;
   MemFileChunk **compchunk_step = &mem_data->reference_current_chunk;
diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 8dc661e6c85..db51bb79f41 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -297,7 +297,7 @@ typedef struct {
   /** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */
   uchar *buf;
   /** Number of bytes used in #WriteData.buf (flushed when exceeded). */
-  int buf_used_len;
+  size_t buf_used_len;
 
 #ifdef USE_WRITE_DATA_LEN
   /** Total number of bytes written. */
@@ -339,12 +339,17 @@ static WriteData *writedata_new(WriteWrap *ww)
   return wd;
 }
 
-static void writedata_do_write(WriteData *wd, const void *mem, int memlen)
+static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen)
 {
   if ((wd == NULL) || wd->error || (mem == NULL) || memlen < 1) {
     return;
   }
 
+  if (memlen > INT_MAX) {
+    BLI_assert(!"Cannot write chunks bigger than INT_MAX.");
+    return;
+  }
+
   if (UNLIKELY(wd->error)) {
     return;
   }
@@ -380,7 +385,7 @@ static void writedata_free(WriteData *wd)
  */
 static void mywrite_flush(WriteData *wd)
 {
-  if (wd->buf_used_len) {
+  if (wd->buf_used_len != 0) {
     writedata_do_write(wd, wd->buf, wd->buf_used_len);
     wd->buf_used_len = 0;
   }
@@ -391,7 +396,7 @@ static void mywrite_flush(WriteData *wd)
  * \param adr: Pointer to new chunk of data
  * \param len: Length of new chunk of data
  */
-static void mywrite(WriteData *wd, const void *adr, int len)
+static void mywrite(WriteData *wd, const void *adr, size_t len)
 {
   if (UNLIKELY(wd->error)) {
     return;
@@ -413,13 +418,13 @@ static void mywrite(WriteData *wd, const void *adr, int len)
     /* if we have a single big chunk, write existing data in
      * buffer and write out big chunk in smaller pieces */
     if (len > MYWRITE_MAX_CHUNK) {
-      if (wd->buf_used_len) {
+      if (wd->buf_used_len != 0) {
         writedata_do_write(wd, wd->buf, wd->buf_used_len);
         wd->buf_used_len = 0;
       }
 
       do {
-        int writelen = MIN2(len, MYWRITE_MAX_CHUNK);
+        size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK);
         writedata_do_write(wd, adr, writelen);
         adr = (const char *)adr + writelen;
         len -= writelen;
@@ -467,7 +472,7 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren
  */
 static bool mywrite_end(WriteData *wd)
 {
-  if (wd->buf_used_len) {
+  if (wd->buf_used_len != 0) {
     writedata_do_write(wd, wd->buf, wd->buf_used_len);
     wd->buf_used_len = 0;
   }
@@ -557,7 +562,7 @@ static void writestruct_at_address_nr(
   }
 
   mywrite(wd, &bh, sizeof(BHead));
-  mywrite(wd, data, bh.len);
+  mywrite(wd, data, (size_t)bh.len);
 }
 
 static void writestruct_nr(
@@ -567,7 +572,7 @@ static void writestruct_nr(
 }
 
 /* do not use for structs */
-static void writedata(WriteData *wd, int filecode, int len, const void *adr)
+static void writedata(WriteData *wd, int filecode, size_t len, const void *adr)
 {
   BHead bh;
 
@@ -575,15 +580,20 @@ static void writedata(WriteData *wd, int filecode, int len, const void *adr)
     return;
   }
 
+  if (len > INT_MAX) {
+    BLI_assert(!"Cannot write chunks bigger than INT_MAX.");
+    return;
+  }
+
   /* align to 4 (writes uninitialized bytes in some cases) */
-  len = (len + 3) & ~3;
+  len = (len + 3) & ~((size_t)3);
 
   /* init BHead */
   bh.code = filecode;
   bh.old = adr;
   bh.nr = 1;
   bh.SDNAnr = 0;
-  bh.len = len;
+  bh.len = (int)len;
 
   mywrite(wd, &bh, sizeof(BHead));
   mywrite(wd, adr, len);
@@ -1679,10 +1689,10 @@ static void write_scene(BlendWriter *writer, Scene *sce, const void *id_address)
   if (sce->r.avicodecdata) {
     BLO_write_struct(writer, AviCodecData, sce->r.avicodecdata);
     if (sce->r.avicodecdata->lpFormat) {
-      BLO_write_raw(writer, sce->r.avicodecdata->cbFormat, sce->r.avicodecdata->lpFormat);
+      BLO_write_raw(writer, (size_t)sce->r.avicodecdata->cbFormat, sce->r.avicodecdata->lpFormat);
     }
     if (sce->r.avicodecdata->lpParms) {
-      BLO_write_raw(writer, sce->r.avicodecdata->cbParms, sce->r.avicodecdata->lpParms);
+      BLO_write_raw(writer, (size_t)sce->r.avicodecdata->cbParms, sce->r.avicodecdata->lpParms);
     }
   }
   if (sce->r.ffcodecdata.properties) {
@@ -1947,7 +1957,7 @@ static void write_area_regions(BlendWriter *writer, ScrArea *area)
       LISTBASE_FOREACH (ConsoleLine *, cl, &con->history) {
         /* 'len_alloc' is invalid on write, set from 'len' on read */
         BLO_write_struct(writer, ConsoleLine, cl);
-        BLO_write_raw(writer, cl->len + 1, cl->line);
+        BLO_write_raw(writer, (size_t)cl->len + 1, cl->line);
       }
       BLO_write_struct(writer, SpaceConsole, sl);
     }
@@ -2411,7 +2421,7 @@ static bool write_file_handle(Main *mainvar,
    *
    * Note that we *borrow* the pointer to 'DNAstr',
    * so writing each time uses the same address and doesn't cause unnecessary undo overhead. */
-  writedata(wd, DNA1, wd->sdna->data_len, wd->sdna->data);
+  writedata(wd, DNA1, (size_t)wd->sdna->data_len, wd->sdna->data);
 
   /* end of file */
   memset(&bhead, 0, sizeof(BHead));
@@ -2632,7 +2642,7 @@ bool BLO_write_file_mem(Main *mainvar, MemFile *compare, MemFile *current, int w
   return (err == 0);
 }
 
-void BLO_write_raw(BlendWriter *writer, int size_in_bytes, const void *data_ptr)
+void BLO_write_raw(BlendWriter *writer, size_t size_in_bytes, const void *data_ptr)
 {
   writedata(writer->wd, DATA, size_in_bytes, data_ptr);
 }
@@ -2708,38 +2718,38 @@ int BLO_get_struct_id_by_name(BlendWriter *writer, const char *struct_name)
   return struct_id;
 }
 
-void BLO_write_int32_array(BlendWriter *writer, int size, const int32_t *data_ptr)
+void BLO_write_int32_array(BlendWriter *writer, uint num, const int32_t *data_ptr)
 {
-  BLO_write_raw(writer, sizeof(int32_t) * size, data_ptr);
+  BLO_write_raw(writer, sizeof(int32_t) * (size_t)num, data_ptr);
 }
 
-void BLO_write_uint32_array(BlendWriter *writer, int size, const uint32_t *data_ptr)
+void BLO_write_uint32_array(BlendWriter *writer, uint num, const uint32_t *data_ptr)
 {
-  BLO_write_raw(writer, sizeof(uint32_t) * size, data_ptr);
+  BLO_write_raw(writer, sizeof(uint32_t) * (size_t)num, data_ptr);
 }
 
-void BLO_write_float_array(BlendWriter *writer, int size, const float *data_ptr)
+void BLO_write_float_array(BlendWriter *writer, uint num, const float *data_ptr)
 {
-  BLO_write_raw(writer, sizeof(float) * size, data_ptr);
+  BLO_write_raw(writer, sizeof(float) * (size_t)num, data_ptr);
 }
 
-void BLO_write_pointer_array(BlendWriter *writer, int size, const void *data_ptr)
+void BLO_write_pointer_array(BlendWriter *writer, uint num, const void *data_ptr)
 {
-  BLO_write_raw(writer, sizeof(void *) * size, data_ptr);
+  BLO_write_raw(writer, sizeof(void *) * (size_t)num, data_ptr);
 }
 
-void BLO_write_float3_array(BlendWriter *writer, int size, const float *data_ptr)
+void BLO_write_float3_array(BlendWriter *writer, uint num, const float *data_ptr)
 {
-  BLO_write_raw(writer, sizeof(float[3]) * size, data_ptr);
+  BLO_write_raw(writer, sizeof(float[3]) * (size_t)num, data_ptr);
 }
 
 /**
  * Write a null terminated string.
  */
-void BLO_write_string(BlendWriter *writer, const char *str)
+void BLO_write_string(BlendWriter *writer, const char *data_ptr)
 {
-  if (str != NULL) {
-    BLO_write_raw(writer, strlen(str) + 1, str);
+  if (data_ptr != NULL) {
+    BLO_write_raw(writer, strlen(data_ptr) + 1, data_ptr);
   }
 }