Cycles: Gracefully handle out-of-memory happening in device vector

Currently only image loading benefits of this and will give magenta color
when image manager detects it's running out of memory.

This isn't ideal solution and can't handle all cases. For example, OOM
killer might kill process before it realized it run out of memory, but
in other cases this could prevent some crashes.

Reviewers: juicyfruit, dingto

Differential Revision: https://developer.blender.org/D1502

3 files changed, 25 insertions, 6 deletions

diff --git a/intern/cycles/device/device_memory.h b/intern/cycles/device/device_memory.h
index ba79f8c88ae..8c32d03135e 100644
--- a/intern/cycles/device/device_memory.h
+++ b/intern/cycles/device/device_memory.h
@@ -211,7 +211,10 @@ public:
 	T *resize(size_t width, size_t height = 0, size_t depth = 0)
 	{
 		data_size = width * ((height == 0)? 1: height) * ((depth == 0)? 1: depth);
-		data.resize(data_size);
+		if(data.resize(data_size) == NULL) {
+			clear();
+			return NULL;
+		}
 		data_width = width;
 		data_height = height;
 		data_depth = depth;
@@ -226,7 +229,9 @@ public:
 	T *copy(T *ptr, size_t width, size_t height = 0, size_t depth = 0)
 	{
 		T *mem = resize(width, height, depth);
-		memcpy(mem, ptr, memory_size());
+		if(mem != NULL) {
+			memcpy(mem, ptr, memory_size());
+		}
 		return mem;
 	}
 
diff --git a/intern/cycles/render/image.cpp b/intern/cycles/render/image.cpp
index 7a846e1dd46..486ae5b597f 100644
--- a/intern/cycles/render/image.cpp
+++ b/intern/cycles/render/image.cpp
@@ -450,6 +450,9 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 
 	/* read RGBA pixels */
 	uchar *pixels = (uchar*)tex_img.resize(width, height, depth);
+	if(pixels == NULL) {
+		return false;
+	}
 	bool cmyk = false;
 
 	if(in) {
@@ -573,6 +576,9 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 
 	/* read RGBA pixels */
 	float *pixels = (float*)tex_img.resize(width, height, depth);
+	if(pixels == NULL) {
+		return false;
+	}
 	bool cmyk = false;
 
 	if(in) {
diff --git a/intern/cycles/util/util_vector.h b/intern/cycles/util/util_vector.h
index ee1f997721d..623436483a0 100644
--- a/intern/cycles/util/util_vector.h
+++ b/intern/cycles/util/util_vector.h
@@ -163,7 +163,7 @@ public:
 		util_aligned_free(data);
 	}
 
-	void resize(size_t newsize)
+	T* resize(size_t newsize)
 	{
 		if(newsize == 0) {
 			clear();
@@ -171,7 +171,12 @@ public:
 		else if(newsize != datasize) {
 			if(newsize > capacity) {
 				T *newdata = (T*)util_aligned_malloc(sizeof(T)*newsize, alignment);
-				if(data) {
+				if(newdata == NULL) {
+					/* Allocation failed, likely out of memory. */
+					clear();
+					return NULL;
+				}
+				else if(data) {
 					memcpy(newdata, data, ((datasize < newsize)? datasize: newsize)*sizeof(T));
 					util_aligned_free(data);
 				}
@@ -180,12 +185,15 @@ public:
 			}
 			datasize = newsize;
 		}
+		return data;
 	}
 
 	void clear()
 	{
-		util_aligned_free(data);
-		data = NULL;
+		if(data != NULL) {
+			util_aligned_free(data);
+			data = NULL;
+		}
 		datasize = 0;
 		capacity = 0;
 	}