libc/stdlib: Use __builtin_mul_overflow for reallocarray and calloc

This built-in function (available in both gcc and clang) is more
efficient and generates shorter code than open-coding the test.

Signed-off-by: Keith Packard <keithp@keithp.com>

3 files changed, 22 insertions, 15 deletions

diff --git a/newlib/libc/stdlib/mallocr.c b/newlib/libc/stdlib/mallocr.c
index 26d1c89cc..9ad720ada 100644
--- a/newlib/libc/stdlib/mallocr.c
+++ b/newlib/libc/stdlib/mallocr.c
@@ -3194,7 +3194,7 @@ Void_t* cALLOc(RARG n, elem_size) RDECL size_t n; size_t elem_size;
   mchunkptr p;
   INTERNAL_SIZE_T csz;
 
-  INTERNAL_SIZE_T sz = n * elem_size;
+  INTERNAL_SIZE_T sz;
 
 #if MORECORE_CLEARS
   mchunkptr oldtop;
@@ -3202,6 +3202,12 @@ Void_t* cALLOc(RARG n, elem_size) RDECL size_t n; size_t elem_size;
 #endif
   Void_t* mem;
 
+  if (__builtin_mul_overflow((INTERNAL_SIZE_T) n, (INTERNAL_SIZE_T) elem_size, &sz))
+  {
+    errno = ENOMEM;
+    return 0;
+  }
+
   /* check if expand_top called, in which case don't need to clear */
 #if MORECORE_CLEARS
   MALLOC_LOCK;
diff --git a/newlib/libc/stdlib/nano-mallocr.c b/newlib/libc/stdlib/nano-mallocr.c
index 13b72c99f..04465eb9e 100644
--- a/newlib/libc/stdlib/nano-mallocr.c
+++ b/newlib/libc/stdlib/nano-mallocr.c
@@ -445,8 +445,16 @@ void nano_cfree(RARG void * ptr)
  * Implement calloc simply by calling malloc and set zero */
 void * nano_calloc(RARG malloc_size_t n, malloc_size_t elem)
 {
-    void * mem = nano_malloc(RCALL n * elem);
-    if (mem != NULL) memset(mem, 0, n * elem);
+    ptrdiff_t bytes;
+    void * mem;
+
+    if (__builtin_mul_overflow (n, elem, &bytes))
+    {
+        RERRNO = ENOMEM;
+        return NULL;
+    }
+    mem = nano_malloc(bytes);
+    if (mem != NULL) memset(mem, 0, bytes);
     return mem;
 }
 #endif /* DEFINE_CALLOC */
diff --git a/newlib/libc/stdlib/reallocarray.c b/newlib/libc/stdlib/reallocarray.c
index 4b6cccb70..d1f63c66b 100644
--- a/newlib/libc/stdlib/reallocarray.c
+++ b/newlib/libc/stdlib/reallocarray.c
@@ -16,27 +16,20 @@
  */
 
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD: head/lib/libc/stdlib/reallocarray.c 282314 2015-05-01 18:32:16Z bapt $");
-
 #include <sys/types.h>
 #include <errno.h>
 #include <stdint.h>
 #include <stdlib.h>
 
-/*
- * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
- * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
- */
-#define MUL_NO_OVERFLOW	((size_t)1 << (sizeof(size_t) * 4))
-
 void *
 reallocarray(void *optr, size_t nmemb, size_t size)
 {
+	ptrdiff_t bytes;
 
-	if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
-	    nmemb > 0 && SIZE_MAX / nmemb < size) {
+	if (__builtin_mul_overflow (nmemb, size, &bytes))
+	{
 		errno = ENOMEM;
-		return (NULL);
+		return NULL;
 	}
-	return (realloc(optr, size * nmemb));
+	return realloc(optr, bytes);
 }