FFmpeg output fixes and small improvement

Most part of this commit fixes issues with FFmpeg output with
currently supported codecs:

- avcodec_encode_video might return zero which doesn't mean error
  happened, but blender will handle this as error and will stop
  rendering to video file.
- Changing output video codec wouldn't update "expert" options
  set for video output which leads to some sideeffects like
  ignored Lossless option for x264 codec.

This fixes allowed to add QTRLE codec easily.

2 files changed, 51 insertions, 20 deletions

diff --git a/source/blender/blenkernel/BKE_writeffmpeg.h b/source/blender/blenkernel/BKE_writeffmpeg.h
index e99dece0f7c..4c42d1e4a6c 100644
--- a/source/blender/blenkernel/BKE_writeffmpeg.h
+++ b/source/blender/blenkernel/BKE_writeffmpeg.h
@@ -74,7 +74,7 @@ void filepath_ffmpeg(char* string, struct RenderData* rd);
 
 extern void ffmpeg_set_preset(struct RenderData *rd, int preset);
 extern void ffmpeg_verify_image_type(struct RenderData *rd, struct ImageFormatData *imf);
-extern void ffmpeg_verify_lossless_format(struct RenderData *rd, struct ImageFormatData *imf);
+extern void ffmpeg_verify_codec_settings(struct RenderData *rd);
 
 extern struct IDProperty *ffmpeg_property_add(struct RenderData *Rd, const char *type, int opt_index, int parent_index);
 extern int ffmpeg_property_add_string(struct RenderData *rd, const char *type, const char *str);
diff --git a/source/blender/blenkernel/intern/writeffmpeg.c b/source/blender/blenkernel/intern/writeffmpeg.c
index 0b043e26ab7..1e2791acfee 100644
--- a/source/blender/blenkernel/intern/writeffmpeg.c
+++ b/source/blender/blenkernel/intern/writeffmpeg.c
@@ -49,6 +49,8 @@
 #  include "AUD_C-API.h"
 #endif
 
+#include "BLI_utildefines.h"
+
 #include "BKE_global.h"
 #include "BKE_idprop.h"
 #include "BKE_main.h"
@@ -76,6 +78,7 @@ static AVFormatContext* outfile = 0;
 static AVStream* video_stream = 0;
 static AVStream* audio_stream = 0;
 static AVFrame* current_frame = 0;
+static int img_convert_fmt = 0;
 static struct SwsContext *img_convert_ctx = 0;
 
 static uint8_t* video_buffer = 0;
@@ -250,7 +253,8 @@ static int write_video_frame(RenderData *rd, int cfra, AVFrame* frame, ReportLis
 
 	outsize = avcodec_encode_video(c, video_buffer, video_buffersize, 
 					   frame);
-	if (outsize != 0) {
+
+	if (outsize > 0) {
 		AVPacket packet;
 		av_init_packet(&packet);
 
@@ -268,14 +272,13 @@ static int write_video_frame(RenderData *rd, int cfra, AVFrame* frame, ReportLis
 		packet.data = video_buffer;
 		packet.size = outsize;
 		ret = av_interleaved_write_frame(outfile, &packet);
-	} else {
-		ret = 0;
+		success = (ret == 0);
+	} else if (outsize < 0) {
+		success = 0;
 	}
 
-	if (ret != 0) {
-		success= 0;
+	if (!success)
 		BKE_report(reports, RPT_ERROR, "Error writing frame.");
-	}
 
 	return success;
 }
@@ -290,8 +293,8 @@ static AVFrame* generate_video_frame(uint8_t* pixels, ReportList *reports)
 	int height = c->height;
 	AVFrame* rgb_frame;
 
-	if (c->pix_fmt != PIX_FMT_BGR32) {
-		rgb_frame = alloc_picture(PIX_FMT_BGR32, width, height);
+	if (c->pix_fmt != img_convert_fmt) {
+		rgb_frame = alloc_picture(img_convert_fmt, width, height);
 		if (!rgb_frame) {
 			BKE_report(reports, RPT_ERROR, "Couldn't allocate temporary frame.");
 			return NULL;
@@ -341,7 +344,7 @@ static AVFrame* generate_video_frame(uint8_t* pixels, ReportList *reports)
 		}
 	}
 
-	if (c->pix_fmt != PIX_FMT_BGR32) {
+	if (c->pix_fmt != img_convert_fmt) {
 		sws_scale(img_convert_ctx, (const uint8_t * const*) rgb_frame->data,
 		          rgb_frame->linesize, 0, c->height,
 		          current_frame->data, current_frame->linesize);
@@ -483,7 +486,9 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 	if (!codec) return NULL;
 	
 	/* Be sure to use the correct pixel format(e.g. RGB, YUV) */
-	
+
+	img_convert_fmt = PIX_FMT_BGR32;
+
 	if (codec->pix_fmts) {
 		c->pix_fmt = codec->pix_fmts[0];
 	} else {
@@ -509,6 +514,13 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 		c->pix_fmt = PIX_FMT_RGB32;
 	}
 
+	if ( codec_id == CODEC_ID_QTRLE ) {
+		if (rd->im_format.planes ==  R_IMF_PLANES_RGBA) {
+			c->pix_fmt = PIX_FMT_ARGB;
+			img_convert_fmt = PIX_FMT_BGRA;
+		}
+	}
+
 	if ((of->oformat->flags & AVFMT_GLOBALHEADER)
 //		|| !strcmp(of->oformat->name, "mp4")
 //	    || !strcmp(of->oformat->name, "mov")
@@ -538,14 +550,26 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 		return NULL;
 	}
 
-	video_buffersize = avpicture_get_size(c->pix_fmt, c->width, c->height);
+	if ( codec_id == CODEC_ID_QTRLE ) {
+		// normally it should be enough to have buffer with actual image size,
+		// but some codecs like QTRLE might store extra information in this buffer,
+		// so it should be a way larger
+
+		// maximum video buffer size is 6-bytes per pixel, plus DPX header size (1664)
+		// (from FFmpeg sources)
+		int size = c->width * c->height;
+		video_buffersize = 7*size + 10000;
+	}
+	else
+		video_buffersize = avpicture_get_size(c->pix_fmt, c->width, c->height);
+
 	video_buffer = (uint8_t*)MEM_mallocN(video_buffersize*sizeof(uint8_t),
 						 "FFMPEG video buffer");
 	
 	current_frame = alloc_picture(c->pix_fmt, c->width, c->height);
 
 	img_convert_ctx = sws_getContext(c->width, c->height,
-					 PIX_FMT_BGR32,
+					 img_convert_fmt,
 					 c->width, c->height,
 					 c->pix_fmt,
 					 SWS_BICUBIC,
@@ -1200,12 +1224,14 @@ int ffmpeg_property_add_string(RenderData *rd, const char * type, const char * s
 	return 1;
 }
 
-static void ffmpeg_set_expert_options(RenderData *rd, int preset)
+static void ffmpeg_set_expert_options(RenderData *rd)
 {
+	int codec_id = rd->ffcodecdata.codec;
+
 	if(rd->ffcodecdata.properties)
 		IDP_FreeProperty(rd->ffcodecdata.properties);
 
-	if(preset == FFMPEG_PRESET_H264) {
+	if(codec_id == CODEC_ID_H264) {
 		/*
 		 * All options here are for x264, but must be set via ffmpeg.
 		 * The names are therefore different - Search for "x264 to FFmpeg option mapping"
@@ -1248,6 +1274,12 @@ static void ffmpeg_set_expert_options(RenderData *rd, int preset)
 		if(rd->ffcodecdata.flags & FFMPEG_LOSSLESS_OUTPUT)
 			ffmpeg_property_add_string(rd, "video", "cqp:0");
 	}
+#if 0	/* disabled for after release */
+	else if(codec_id == CODEC_ID_DNXHD) {
+		if(rd->ffcodecdata.flags & FFMPEG_LOSSLESS_OUTPUT)
+			ffmpeg_property_add_string(rd, "video", "mbd:rd");
+	}
+#endif
 }
 
 void ffmpeg_set_preset(RenderData *rd, int preset)
@@ -1317,7 +1349,6 @@ void ffmpeg_set_preset(RenderData *rd, int preset)
 		rd->ffcodecdata.mux_packet_size = 2048;
 		rd->ffcodecdata.mux_rate = 10080000;
 
-		ffmpeg_set_expert_options(rd, preset);
 		break;
 
 	case FFMPEG_PRESET_THEORA:
@@ -1341,6 +1372,8 @@ void ffmpeg_set_preset(RenderData *rd, int preset)
 		break;
 
 	}
+
+	ffmpeg_set_expert_options(rd);
 }
 
 void ffmpeg_verify_image_type(RenderData *rd, ImageFormatData *imf)
@@ -1388,11 +1421,9 @@ void ffmpeg_verify_image_type(RenderData *rd, ImageFormatData *imf)
 	}
 }
 
-void ffmpeg_verify_lossless_format(RenderData *rd, ImageFormatData *imf)
+void ffmpeg_verify_codec_settings(RenderData *rd)
 {
-	if(imf->imtype == R_IMF_IMTYPE_H264) {
-		ffmpeg_set_expert_options(rd, FFMPEG_PRESET_H264);
-	}
+	ffmpeg_set_expert_options(rd);
 }
 
 #endif