Memory: add MEM_malloc_arrayN() function to protect against overflow.

Differential Revision: https://developer.blender.org/D3002

7 files changed, 141 insertions, 0 deletions

diff --git a/intern/guardedalloc/CMakeLists.txt b/intern/guardedalloc/CMakeLists.txt
index 1d041ba5380..10ed4287185 100644
--- a/intern/guardedalloc/CMakeLists.txt
+++ b/intern/guardedalloc/CMakeLists.txt
@@ -38,6 +38,7 @@ set(SRC
 	./intern/mallocn_lockfree_impl.c
 
 	MEM_guardedalloc.h
+	./intern/mallocn_inline.h
 	./intern/mallocn_intern.h
 
 	# only so the header is known by cmake
diff --git a/intern/guardedalloc/MEM_guardedalloc.h b/intern/guardedalloc/MEM_guardedalloc.h
index f6048a04cf3..a57921223e2 100644
--- a/intern/guardedalloc/MEM_guardedalloc.h
+++ b/intern/guardedalloc/MEM_guardedalloc.h
@@ -114,12 +114,26 @@ extern "C" {
 	extern void *(*MEM_callocN)(size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
 
 	/**
+	 * Allocate a block of memory of size (len * size), with tag name
+	 * str, aborting in case of integer overflows to prevent vulnerabilities.
+	 * The memory is cleared. The name must be static, because only a
+	 * pointer to it is stored ! */
+	extern void *(*MEM_calloc_arrayN)(size_t len, size_t size, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
+
+	/**
 	 * Allocate a block of memory of size len, with tag name str. The
 	 * name must be a static, because only a pointer to it is stored !
 	 * */
 	extern void *(*MEM_mallocN)(size_t len, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
 
 	/**
+	 * Allocate a block of memory of size (len * size), with tag name str,
+	 * aborting in case of integer overflow to prevent vulnerabilities. The
+	 * name must be a static, because only a pointer to it is stored !
+	 * */
+	extern void *(*MEM_malloc_arrayN)(size_t len, size_t size, const char *str) /* ATTR_MALLOC */ ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
+
+	/**
 	 * Allocate an aligned block of memory of size len, with tag name str. The
 	 * name must be a static, because only a pointer to it is stored !
 	 * */
diff --git a/intern/guardedalloc/intern/mallocn.c b/intern/guardedalloc/intern/mallocn.c
index 1fd85a0c9f5..b973037358b 100644
--- a/intern/guardedalloc/intern/mallocn.c
+++ b/intern/guardedalloc/intern/mallocn.c
@@ -43,7 +43,9 @@ void *(*MEM_dupallocN)(const void *vmemh) = MEM_lockfree_dupallocN;
 void *(*MEM_reallocN_id)(void *vmemh, size_t len, const char *str) = MEM_lockfree_reallocN_id;
 void *(*MEM_recallocN_id)(void *vmemh, size_t len, const char *str) = MEM_lockfree_recallocN_id;
 void *(*MEM_callocN)(size_t len, const char *str) = MEM_lockfree_callocN;
+void *(*MEM_calloc_arrayN)(size_t len, size_t size, const char *str) = MEM_lockfree_calloc_arrayN;
 void *(*MEM_mallocN)(size_t len, const char *str) = MEM_lockfree_mallocN;
+void *(*MEM_malloc_arrayN)(size_t len, size_t size, const char *str) = MEM_lockfree_malloc_arrayN;
 void *(*MEM_mallocN_aligned)(size_t len, size_t alignment, const char *str) = MEM_lockfree_mallocN_aligned;
 void *(*MEM_mapallocN)(size_t len, const char *str) = MEM_lockfree_mapallocN;
 void (*MEM_printmemlist_pydict)(void) = MEM_lockfree_printmemlist_pydict;
@@ -107,7 +109,9 @@ void MEM_use_guarded_allocator(void)
 	MEM_reallocN_id = MEM_guarded_reallocN_id;
 	MEM_recallocN_id = MEM_guarded_recallocN_id;
 	MEM_callocN = MEM_guarded_callocN;
+	MEM_calloc_arrayN = MEM_guarded_calloc_arrayN;
 	MEM_mallocN = MEM_guarded_mallocN;
+	MEM_malloc_arrayN = MEM_guarded_malloc_arrayN;
 	MEM_mallocN_aligned = MEM_guarded_mallocN_aligned;
 	MEM_mapallocN = MEM_guarded_mapallocN;
 	MEM_printmemlist_pydict = MEM_guarded_printmemlist_pydict;
diff --git a/intern/guardedalloc/intern/mallocn_guarded_impl.c b/intern/guardedalloc/intern/mallocn_guarded_impl.c
index 76b7e072321..d012cce5334 100644
--- a/intern/guardedalloc/intern/mallocn_guarded_impl.c
+++ b/intern/guardedalloc/intern/mallocn_guarded_impl.c
@@ -542,6 +542,21 @@ void *MEM_guarded_mallocN(size_t len, const char *str)
 	return NULL;
 }
 
+void *MEM_guarded_malloc_arrayN(size_t len, size_t size, const char *str)
+{
+	size_t total_size;
+	if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
+		print_error("Malloc array aborted due to integer overflow: "
+		            "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
+		            SIZET_ARG(len), SIZET_ARG(size), str,
+		            (unsigned int) mem_in_use);
+		abort();
+		return NULL;
+	}
+
+	return MEM_guarded_mallocN(total_size, str);
+}
+
 void *MEM_guarded_mallocN_aligned(size_t len, size_t alignment, const char *str)
 {
 	MemHead *memh;
@@ -612,6 +627,21 @@ void *MEM_guarded_callocN(size_t len, const char *str)
 	return NULL;
 }
 
+void *MEM_guarded_calloc_arrayN(size_t len, size_t size, const char *str)
+{
+	size_t total_size;
+	if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
+		print_error("Calloc array aborted due to integer overflow: "
+		            "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
+		            SIZET_ARG(len), SIZET_ARG(size), str,
+		            (unsigned int) mem_in_use);
+		abort();
+		return NULL;
+	}
+
+	return MEM_guarded_callocN(total_size, str);
+}
+
 /* note; mmap returns zero'd memory */
 void *MEM_guarded_mapallocN(size_t len, const char *str)
 {
diff --git a/intern/guardedalloc/intern/mallocn_inline.h b/intern/guardedalloc/intern/mallocn_inline.h
new file mode 100644
index 00000000000..7f0fa2bd388
--- /dev/null
+++ b/intern/guardedalloc/intern/mallocn_inline.h
@@ -0,0 +1,56 @@
+/*
+ * Adapted from jemalloc, to protect against buffer overflow vulnerabilities.
+ *
+ * Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
+ * All rights reserved.
+ * Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
+ * Copyright (C) 2009-2017 Facebook, Inc.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright notice(s),
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice(s),
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file guardedalloc/intern/mallocn_inline.h
+ *  \ingroup MEM
+ */
+
+#ifndef __MALLOCN_INLINE_H__
+#define __MALLOCN_INLINE_H__
+
+MEM_INLINE bool MEM_size_safe_multiply(size_t a, size_t b, size_t *result)
+{
+	/* A size_t with its high-half bits all set to 1. */
+	const size_t high_bits = SIZE_MAX << (sizeof(size_t) * 8 / 2);
+	*result = a * b;
+
+	if (UNLIKELY(*result == 0)) {
+		return (a == 0 || b == 0);
+	}
+
+	/*
+	 * We got a non-zero size, but we don't know if we overflowed to get
+	 * there.  To avoid having to do a divide, we'll be clever and note that
+	 * if both A and B can be represented in N/2 bits, then their product
+	 * can be represented in N bits (without the possibility of overflow).
+	 */
+	return ((high_bits & (a | b)) == 0 || (*result / b == a));
+}
+
+#endif  /* __MALLOCN_INLINE_H__ */
+
diff --git a/intern/guardedalloc/intern/mallocn_intern.h b/intern/guardedalloc/intern/mallocn_intern.h
index 9a5848c50ad..4b2282ae9ad 100644
--- a/intern/guardedalloc/intern/mallocn_intern.h
+++ b/intern/guardedalloc/intern/mallocn_intern.h
@@ -115,6 +115,8 @@ size_t malloc_usable_size(void *ptr);
 /* Real pointer returned by the malloc or aligned_alloc. */
 #define MEMHEAD_REAL_PTR(memh) ((char *)memh - MEMHEAD_ALIGN_PADDING(memh->alignment))
 
+#include "mallocn_inline.h"
+
 void *aligned_malloc(size_t size, size_t alignment);
 void aligned_free(void *ptr);
 
@@ -125,7 +127,9 @@ void *MEM_lockfree_dupallocN(const void *vmemh) ATTR_MALLOC ATTR_WARN_UNUSED_RES
 void *MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
 void *MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
 void *MEM_lockfree_callocN(size_t len, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *UNUSED(str))  ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
 void *MEM_lockfree_mallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
 void *MEM_lockfree_mallocN_aligned(size_t len, size_t alignment, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(3);
 void *MEM_lockfree_mapallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
 void MEM_lockfree_printmemlist_pydict(void);
@@ -152,7 +156,9 @@ void *MEM_guarded_dupallocN(const void *vmemh) ATTR_MALLOC ATTR_WARN_UNUSED_RESU
 void *MEM_guarded_reallocN_id(void *vmemh, size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
 void *MEM_guarded_recallocN_id(void *vmemh, size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(2);
 void *MEM_guarded_callocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_guarded_calloc_arrayN(size_t len, size_t size, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
 void *MEM_guarded_mallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
+void *MEM_guarded_malloc_arrayN(size_t len, size_t size, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1,2) ATTR_NONNULL(3);
 void *MEM_guarded_mallocN_aligned(size_t len, size_t alignment, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(3);
 void *MEM_guarded_mapallocN(size_t len, const char *UNUSED(str)) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2);
 void MEM_guarded_printmemlist_pydict(void);
diff --git a/intern/guardedalloc/intern/mallocn_lockfree_impl.c b/intern/guardedalloc/intern/mallocn_lockfree_impl.c
index 66573b91ace..2899a37a1b2 100644
--- a/intern/guardedalloc/intern/mallocn_lockfree_impl.c
+++ b/intern/guardedalloc/intern/mallocn_lockfree_impl.c
@@ -293,6 +293,21 @@ void *MEM_lockfree_callocN(size_t len, const char *str)
 	return NULL;
 }
 
+void *MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *str)
+{
+	size_t total_size;
+	if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
+		print_error("Calloc array aborted due to integer overflow: "
+		            "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
+		            SIZET_ARG(len), SIZET_ARG(size), str,
+		            (unsigned int) mem_in_use);
+		abort();
+		return NULL;
+	}
+
+	return MEM_lockfree_callocN(total_size, str);
+}
+
 void *MEM_lockfree_mallocN(size_t len, const char *str)
 {
 	MemHead *memh;
@@ -318,6 +333,21 @@ void *MEM_lockfree_mallocN(size_t len, const char *str)
 	return NULL;
 }
 
+void *MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *str)
+{
+	size_t total_size;
+	if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
+		print_error("Malloc array aborted due to integer overflow: "
+		            "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
+		            SIZET_ARG(len), SIZET_ARG(size), str,
+		            (unsigned int) mem_in_use);
+		abort();
+		return NULL;
+	}
+
+	return MEM_lockfree_mallocN(total_size, str);
+}
+
 void *MEM_lockfree_mallocN_aligned(size_t len, size_t alignment, const char *str)
 {
 	MemHeadAligned *memh;