Fix T44616: Cycles crashes loading 42k by 21k textures

Simple integer overflow issue.

TODO(sergey): Check on CPU cubic sampling, it might also need size_t.

2 files changed, 29 insertions, 23 deletions

diff --git a/intern/cycles/blender/blender_session.cpp b/intern/cycles/blender/blender_session.cpp
index 897514668f6..bb345e2b3cf 100644
--- a/intern/cycles/blender/blender_session.cpp
+++ b/intern/cycles/blender/blender_session.cpp
@@ -1020,18 +1020,19 @@ bool BlenderSession::builtin_image_pixels(const string &builtin_name, void *buil
 
 	unsigned char *image_pixels;
 	image_pixels = image_get_pixels_for_frame(b_image, frame);
+	size_t num_pixels = ((size_t)width) * height;
 
 	if(image_pixels) {
-		memcpy(pixels, image_pixels, width * height * channels * sizeof(unsigned char));
+		memcpy(pixels, image_pixels, num_pixels * channels * sizeof(unsigned char));
 		MEM_freeN(image_pixels);
 	}
 	else {
 		if(channels == 1) {
-			memset(pixels, 0, width * height * sizeof(unsigned char));
+			memset(pixels, 0, num_pixels * sizeof(unsigned char));
 		}
 		else {
 			unsigned char *cp = pixels;
-			for(int i = 0; i < width * height; i++, cp += channels) {
+			for(size_t i = 0; i < num_pixels; i++, cp += channels) {
 				cp[0] = 255;
 				cp[1] = 0;
 				cp[2] = 255;
@@ -1043,7 +1044,7 @@ bool BlenderSession::builtin_image_pixels(const string &builtin_name, void *buil
 
 	/* premultiply, byte images are always straight for blender */
 	unsigned char *cp = pixels;
-	for(int i = 0; i < width * height; i++, cp += channels) {
+	for(size_t i = 0; i < num_pixels; i++, cp += channels) {
 		cp[0] = (cp[0] * cp[3]) >> 8;
 		cp[1] = (cp[1] * cp[3]) >> 8;
 		cp[2] = (cp[2] * cp[3]) >> 8;
@@ -1072,18 +1073,19 @@ bool BlenderSession::builtin_image_float_pixels(const string &builtin_name, void
 
 		float *image_pixels;
 		image_pixels = image_get_float_pixels_for_frame(b_image, frame);
+		size_t num_pixels = ((size_t)width) * height;
 
 		if(image_pixels) {
-			memcpy(pixels, image_pixels, width * height * channels * sizeof(float));
+			memcpy(pixels, image_pixels, num_pixels * channels * sizeof(float));
 			MEM_freeN(image_pixels);
 		}
 		else {
 			if(channels == 1) {
-				memset(pixels, 0, width * height * sizeof(float));
+				memset(pixels, 0, num_pixels * sizeof(float));
 			}
 			else {
 				float *fp = pixels;
-				for(int i = 0; i < width * height; i++, fp += channels) {
+				for(int i = 0; i < num_pixels; i++, fp += channels) {
 					fp[0] = 1.0f;
 					fp[1] = 0.0f;
 					fp[2] = 1.0f;
@@ -1109,11 +1111,12 @@ bool BlenderSession::builtin_image_float_pixels(const string &builtin_name, void
 		int width = resolution.x * amplify;
 		int height = resolution.y * amplify;
 		int depth = resolution.z * amplify;
+		size_t num_pixels = ((size_t)width) * height * depth;
 
 		if(builtin_name == Attribute::standard_name(ATTR_STD_VOLUME_DENSITY)) {
 			SmokeDomainSettings_density_grid_get_length(&b_domain.ptr, &length);
 
-			if(length == width*height*depth) {
+			if(length == num_pixels) {
 				SmokeDomainSettings_density_grid_get(&b_domain.ptr, pixels);
 				return true;
 			}
@@ -1123,7 +1126,7 @@ bool BlenderSession::builtin_image_float_pixels(const string &builtin_name, void
 			 * as 1500..3000 K with the first part faded to zero density */
 			SmokeDomainSettings_flame_grid_get_length(&b_domain.ptr, &length);
 
-			if(length == width*height*depth) {
+			if(length == num_pixels) {
 				SmokeDomainSettings_flame_grid_get(&b_domain.ptr, pixels);
 				return true;
 			}
@@ -1132,7 +1135,7 @@ bool BlenderSession::builtin_image_float_pixels(const string &builtin_name, void
 			/* the RGB is "premultiplied" by density for better interpolation results */
 			SmokeDomainSettings_color_grid_get_length(&b_domain.ptr, &length);
 
-			if(length == width*height*depth*4) {
+			if(length == num_pixels*4) {
 				SmokeDomainSettings_color_grid_get(&b_domain.ptr, pixels);
 				return true;
 			}
diff --git a/intern/cycles/render/image.cpp b/intern/cycles/render/image.cpp
index f0608965d09..c62afcd7719 100644
--- a/intern/cycles/render/image.cpp
+++ b/intern/cycles/render/image.cpp
@@ -407,7 +407,7 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 			int scanlinesize = width*components*sizeof(uchar);
 
 			in->read_image(TypeDesc::UINT8,
-				(uchar*)pixels + (height-1)*scanlinesize,
+				(uchar*)pixels + (((size_t)height)-1)*scanlinesize,
 				AutoStride,
 				-scanlinesize,
 				AutoStride);
@@ -425,9 +425,10 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 		builtin_image_pixels_cb(img->filename, img->builtin_data, pixels);
 	}
 
+	size_t num_pixels = ((size_t)width) * height * depth;
 	if(cmyk) {
 		/* CMYK */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+2] = (pixels[i*4+2]*pixels[i*4+3])/255;
 			pixels[i*4+1] = (pixels[i*4+1]*pixels[i*4+3])/255;
 			pixels[i*4+0] = (pixels[i*4+0]*pixels[i*4+3])/255;
@@ -436,7 +437,7 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 	}
 	else if(components == 2) {
 		/* grayscale + alpha */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = pixels[i*2+1];
 			pixels[i*4+2] = pixels[i*2+0];
 			pixels[i*4+1] = pixels[i*2+0];
@@ -445,7 +446,7 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 	}
 	else if(components == 3) {
 		/* RGB */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 255;
 			pixels[i*4+2] = pixels[i*3+2];
 			pixels[i*4+1] = pixels[i*3+1];
@@ -454,7 +455,7 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 	}
 	else if(components == 1) {
 		/* grayscale */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 255;
 			pixels[i*4+2] = pixels[i];
 			pixels[i*4+1] = pixels[i];
@@ -463,7 +464,7 @@ bool ImageManager::file_load_image(Image *img, device_vector<uchar4>& tex_img)
 	}
 
 	if(img->use_alpha == false) {
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 255;
 		}
 	}
@@ -529,7 +530,7 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 		vector<float> tmppixels;
 
 		if(components > 4) {
-			tmppixels.resize(width*height*components);
+			tmppixels.resize(((size_t)width)*height*components);
 			readpixels = &tmppixels[0];
 		}
 
@@ -547,7 +548,8 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 		}
 
 		if(components > 4) {
-			for(int i = width*height-1; i >= 0; i--) {
+			size_t dimensions = ((size_t)width)*height;
+			for(size_t i = dimensions-1, pixel = 0; pixel < dimensions; pixel++, i--) {
 				pixels[i*4+3] = tmppixels[i*components+3];
 				pixels[i*4+2] = tmppixels[i*components+2];
 				pixels[i*4+1] = tmppixels[i*components+1];
@@ -566,9 +568,10 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 		builtin_image_float_pixels_cb(img->filename, img->builtin_data, pixels);
 	}
 
+	size_t num_pixels = ((size_t)width) * height * depth;
 	if(cmyk) {
 		/* CMYK */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 255;
 			pixels[i*4+2] = (pixels[i*4+2]*pixels[i*4+3])/255;
 			pixels[i*4+1] = (pixels[i*4+1]*pixels[i*4+3])/255;
@@ -577,7 +580,7 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 	}
 	else if(components == 2) {
 		/* grayscale + alpha */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = pixels[i*2+1];
 			pixels[i*4+2] = pixels[i*2+0];
 			pixels[i*4+1] = pixels[i*2+0];
@@ -586,7 +589,7 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 	}
 	else if(components == 3) {
 		/* RGB */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 1.0f;
 			pixels[i*4+2] = pixels[i*3+2];
 			pixels[i*4+1] = pixels[i*3+1];
@@ -595,7 +598,7 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 	}
 	else if(components == 1) {
 		/* grayscale */
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 1.0f;
 			pixels[i*4+2] = pixels[i];
 			pixels[i*4+1] = pixels[i];
@@ -604,7 +607,7 @@ bool ImageManager::file_load_float_image(Image *img, device_vector<float4>& tex_
 	}
 
 	if(img->use_alpha == false) {
-		for(int i = width*height*depth-1; i >= 0; i--) {
+		for(size_t i = num_pixels-1, pixel = 0; pixel < num_pixels; pixel++, i--) {
 			pixels[i*4+3] = 1.0f;
 		}
 	}