FFmpeg output fixes and small improvement

Most part of this commit fixes issues with FFmpeg output with
currently supported codecs:

- avcodec_encode_video might return zero which doesn't mean error
  happened, but blender will handle this as error and will stop
  rendering to video file.
- Changing output video codec wouldn't update "expert" options
  set for video output which leads to some sideeffects like
  ignored Lossless option for x264 codec.

This fixes allowed to add QTRLE codec easily.

3 files changed, 81 insertions, 25 deletions

diff --git a/source/blender/blenkernel/BKE_writeffmpeg.h b/source/blender/blenkernel/BKE_writeffmpeg.h
index e99dece0f7c..4c42d1e4a6c 100644
--- a/source/blender/blenkernel/BKE_writeffmpeg.h
+++ b/source/blender/blenkernel/BKE_writeffmpeg.h
@@ -74,7 +74,7 @@ void filepath_ffmpeg(char* string, struct RenderData* rd);
 
 extern void ffmpeg_set_preset(struct RenderData *rd, int preset);
 extern void ffmpeg_verify_image_type(struct RenderData *rd, struct ImageFormatData *imf);
-extern void ffmpeg_verify_lossless_format(struct RenderData *rd, struct ImageFormatData *imf);
+extern void ffmpeg_verify_codec_settings(struct RenderData *rd);
 
 extern struct IDProperty *ffmpeg_property_add(struct RenderData *Rd, const char *type, int opt_index, int parent_index);
 extern int ffmpeg_property_add_string(struct RenderData *rd, const char *type, const char *str);
diff --git a/source/blender/blenkernel/intern/writeffmpeg.c b/source/blender/blenkernel/intern/writeffmpeg.c
index 0b043e26ab7..1e2791acfee 100644
--- a/source/blender/blenkernel/intern/writeffmpeg.c
+++ b/source/blender/blenkernel/intern/writeffmpeg.c
@@ -49,6 +49,8 @@
 #  include "AUD_C-API.h"
 #endif
 
+#include "BLI_utildefines.h"
+
 #include "BKE_global.h"
 #include "BKE_idprop.h"
 #include "BKE_main.h"
@@ -76,6 +78,7 @@ static AVFormatContext* outfile = 0;
 static AVStream* video_stream = 0;
 static AVStream* audio_stream = 0;
 static AVFrame* current_frame = 0;
+static int img_convert_fmt = 0;
 static struct SwsContext *img_convert_ctx = 0;
 
 static uint8_t* video_buffer = 0;
@@ -250,7 +253,8 @@ static int write_video_frame(RenderData *rd, int cfra, AVFrame* frame, ReportLis
 
 	outsize = avcodec_encode_video(c, video_buffer, video_buffersize, 
 					   frame);
-	if (outsize != 0) {
+
+	if (outsize > 0) {
 		AVPacket packet;
 		av_init_packet(&packet);
 
@@ -268,14 +272,13 @@ static int write_video_frame(RenderData *rd, int cfra, AVFrame* frame, ReportLis
 		packet.data = video_buffer;
 		packet.size = outsize;
 		ret = av_interleaved_write_frame(outfile, &packet);
-	} else {
-		ret = 0;
+		success = (ret == 0);
+	} else if (outsize < 0) {
+		success = 0;
 	}
 
-	if (ret != 0) {
-		success= 0;
+	if (!success)
 		BKE_report(reports, RPT_ERROR, "Error writing frame.");
-	}
 
 	return success;
 }
@@ -290,8 +293,8 @@ static AVFrame* generate_video_frame(uint8_t* pixels, ReportList *reports)
 	int height = c->height;
 	AVFrame* rgb_frame;
 
-	if (c->pix_fmt != PIX_FMT_BGR32) {
-		rgb_frame = alloc_picture(PIX_FMT_BGR32, width, height);
+	if (c->pix_fmt != img_convert_fmt) {
+		rgb_frame = alloc_picture(img_convert_fmt, width, height);
 		if (!rgb_frame) {
 			BKE_report(reports, RPT_ERROR, "Couldn't allocate temporary frame.");
 			return NULL;
@@ -341,7 +344,7 @@ static AVFrame* generate_video_frame(uint8_t* pixels, ReportList *reports)
 		}
 	}
 
-	if (c->pix_fmt != PIX_FMT_BGR32) {
+	if (c->pix_fmt != img_convert_fmt) {
 		sws_scale(img_convert_ctx, (const uint8_t * const*) rgb_frame->data,
 		          rgb_frame->linesize, 0, c->height,
 		          current_frame->data, current_frame->linesize);
@@ -483,7 +486,9 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 	if (!codec) return NULL;
 	
 	/* Be sure to use the correct pixel format(e.g. RGB, YUV) */
-	
+
+	img_convert_fmt = PIX_FMT_BGR32;
+
 	if (codec->pix_fmts) {
 		c->pix_fmt = codec->pix_fmts[0];
 	} else {
@@ -509,6 +514,13 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 		c->pix_fmt = PIX_FMT_RGB32;
 	}
 
+	if ( codec_id == CODEC_ID_QTRLE ) {
+		if (rd->im_format.planes ==  R_IMF_PLANES_RGBA) {
+			c->pix_fmt = PIX_FMT_ARGB;
+			img_convert_fmt = PIX_FMT_BGRA;
+		}
+	}
+
 	if ((of->oformat->flags & AVFMT_GLOBALHEADER)
 //		|| !strcmp(of->oformat->name, "mp4")
 //	    || !strcmp(of->oformat->name, "mov")
@@ -538,14 +550,26 @@ static AVStream* alloc_video_stream(RenderData *rd, int codec_id, AVFormatContex
 		return NULL;
 	}
 
-	video_buffersize = avpicture_get_size(c->pix_fmt, c->width, c->height);
+	if ( codec_id == CODEC_ID_QTRLE ) {
+		// normally it should be enough to have buffer with actual image size,
+		// but some codecs like QTRLE might store extra information in this buffer,
+		// so it should be a way larger
+
+		// maximum video buffer size is 6-bytes per pixel, plus DPX header size (1664)
+		// (from FFmpeg sources)
+		int size = c->width * c->height;
+		video_buffersize = 7*size + 10000;
+	}
+	else
+		video_buffersize = avpicture_get_size(c->pix_fmt, c->width, c->height);
+
 	video_buffer = (uint8_t*)MEM_mallocN(video_buffersize*sizeof(uint8_t),
 						 "FFMPEG video buffer");
 	
 	current_frame = alloc_picture(c->pix_fmt, c->width, c->height);
 
 	img_convert_ctx = sws_getContext(c->width, c->height,
-					 PIX_FMT_BGR32,
+					 img_convert_fmt,
 					 c->width, c->height,
 					 c->pix_fmt,
 					 SWS_BICUBIC,
@@ -1200,12 +1224,14 @@ int ffmpeg_property_add_string(RenderData *rd, const char * type, const char * s
 	return 1;
 }
 
-static void ffmpeg_set_expert_options(RenderData *rd, int preset)
+static void ffmpeg_set_expert_options(RenderData *rd)
 {
+	int codec_id = rd->ffcodecdata.codec;
+
 	if(rd->ffcodecdata.properties)
 		IDP_FreeProperty(rd->ffcodecdata.properties);
 
-	if(preset == FFMPEG_PRESET_H264) {
+	if(codec_id == CODEC_ID_H264) {
 		/*
 		 * All options here are for x264, but must be set via ffmpeg.
 		 * The names are therefore different - Search for "x264 to FFmpeg option mapping"
@@ -1248,6 +1274,12 @@ static void ffmpeg_set_expert_options(RenderData *rd, int preset)
 		if(rd->ffcodecdata.flags & FFMPEG_LOSSLESS_OUTPUT)
 			ffmpeg_property_add_string(rd, "video", "cqp:0");
 	}
+#if 0	/* disabled for after release */
+	else if(codec_id == CODEC_ID_DNXHD) {
+		if(rd->ffcodecdata.flags & FFMPEG_LOSSLESS_OUTPUT)
+			ffmpeg_property_add_string(rd, "video", "mbd:rd");
+	}
+#endif
 }
 
 void ffmpeg_set_preset(RenderData *rd, int preset)
@@ -1317,7 +1349,6 @@ void ffmpeg_set_preset(RenderData *rd, int preset)
 		rd->ffcodecdata.mux_packet_size = 2048;
 		rd->ffcodecdata.mux_rate = 10080000;
 
-		ffmpeg_set_expert_options(rd, preset);
 		break;
 
 	case FFMPEG_PRESET_THEORA:
@@ -1341,6 +1372,8 @@ void ffmpeg_set_preset(RenderData *rd, int preset)
 		break;
 
 	}
+
+	ffmpeg_set_expert_options(rd);
 }
 
 void ffmpeg_verify_image_type(RenderData *rd, ImageFormatData *imf)
@@ -1388,11 +1421,9 @@ void ffmpeg_verify_image_type(RenderData *rd, ImageFormatData *imf)
 	}
 }
 
-void ffmpeg_verify_lossless_format(RenderData *rd, ImageFormatData *imf)
+void ffmpeg_verify_codec_settings(RenderData *rd)
 {
-	if(imf->imtype == R_IMF_IMTYPE_H264) {
-		ffmpeg_set_expert_options(rd, FFMPEG_PRESET_H264);
-	}
+	ffmpeg_set_expert_options(rd);
 }
 
 #endif
diff --git a/source/blender/makesrna/intern/rna_scene.c b/source/blender/makesrna/intern/rna_scene.c
index 290552d394f..fe601b9bd19 100644
--- a/source/blender/makesrna/intern/rna_scene.c
+++ b/source/blender/makesrna/intern/rna_scene.c
@@ -710,7 +710,21 @@ static EnumPropertyItem *rna_ImageFormatSettings_color_mode_itemf(bContext *C, P
 	 * where 'BW' will force greyscale even if the output format writes
 	 * as RGBA, this is age old blender convention and not sure how useful
 	 * it really is but keep it for now - campbell */
-	const char chan_flag= BKE_imtype_valid_channels(imf->imtype) | (is_render ? IMA_CHAN_FLAG_BW : 0);
+	char chan_flag= BKE_imtype_valid_channels(imf->imtype) | (is_render ? IMA_CHAN_FLAG_BW : 0);
+
+#ifdef WITH_FFMPEG
+	/* a WAY more crappy case than B&W flag: depending on codec, file format MIGHT support
+	 * alpha channel. for example MPEG format with h264 codec can't do alpha channel, but
+	 * the same MPEG format with QTRLE codec can easily handle alpga channel.
+	 * not sure how to deal with such cases in a nicer way (sergey) */
+	if(is_render) {
+		Scene *scene = ptr->id.data;
+		RenderData *rd = &scene->r;
+
+		if (rd->ffcodecdata.codec == CODEC_ID_QTRLE)
+			chan_flag |= IMA_CHAN_FLAG_ALPHA;
+	}
+#endif
 
 	if (chan_flag == (IMA_CHAN_FLAG_BW|IMA_CHAN_FLAG_RGB|IMA_CHAN_FLAG_ALPHA)) {
 		return image_color_mode_items;
@@ -897,7 +911,16 @@ static void rna_FFmpegSettings_lossless_output_set(PointerRNA *ptr, int value)
 		rd->ffcodecdata.flags |= FFMPEG_LOSSLESS_OUTPUT;
 	else
 		rd->ffcodecdata.flags &= ~FFMPEG_LOSSLESS_OUTPUT;
-	ffmpeg_verify_lossless_format(rd, &rd->im_format);
+
+	ffmpeg_verify_codec_settings(rd);
+}
+
+static void rna_FFmpegSettings_codec_settings_update(Main *UNUSED(bmain), Scene *UNUSED(scene_unused), PointerRNA *ptr)
+{
+	Scene *scene = (Scene *) ptr->id.data;
+	RenderData *rd = &scene->r;
+
+	ffmpeg_verify_codec_settings(rd);
 }
 #endif
 
@@ -2809,6 +2832,8 @@ static void rna_def_scene_ffmpeg_settings(BlenderRNA *brna)
 		{CODEC_ID_THEORA, "THEORA", 0, "Theora", ""},
 		{CODEC_ID_FLV1, "FLASH", 0, "Flash Video", ""},
 		{CODEC_ID_FFV1, "FFV1", 0, "FFmpeg video codec #1", ""},
+		{CODEC_ID_QTRLE, "QTRLE", 0, "QTRLE", ""},
+		/* {CODEC_ID_DNXHD, "DNXHD", 0, "DNxHD", ""},*/ /* disabled for after release */
 		{0, NULL, 0, NULL, NULL}};
 
 	static EnumPropertyItem ffmpeg_audio_codec_items[] = {
@@ -2840,17 +2865,17 @@ static void rna_def_scene_ffmpeg_settings(BlenderRNA *brna)
 	RNA_def_property_enum_bitflag_sdna(prop, NULL, "type");
 	RNA_def_property_enum_items(prop, ffmpeg_format_items);
 	RNA_def_property_ui_text(prop, "Format", "Output file format");
-	RNA_def_property_update(prop, NC_SCENE|ND_RENDER_OPTIONS, NULL);
+	RNA_def_property_update(prop, NC_SCENE|ND_RENDER_OPTIONS, "rna_FFmpegSettings_codec_settings_update");
 
 	prop = RNA_def_property(srna, "codec", PROP_ENUM, PROP_NONE);
 	RNA_def_property_enum_bitflag_sdna(prop, NULL, "codec");
 	RNA_def_property_enum_items(prop, ffmpeg_codec_items);
 	RNA_def_property_ui_text(prop, "Codec", "FFmpeg codec to use");
-	RNA_def_property_update(prop, NC_SCENE|ND_RENDER_OPTIONS, NULL);
+	RNA_def_property_update(prop, NC_SCENE|ND_RENDER_OPTIONS, "rna_FFmpegSettings_codec_settings_update");
 
 	prop = RNA_def_property(srna, "video_bitrate", PROP_INT, PROP_NONE);
 	RNA_def_property_int_sdna(prop, NULL, "video_bitrate");
-	RNA_def_property_range(prop, 1, 14000);
+	RNA_def_property_range(prop, 1, 220000);
 	RNA_def_property_ui_text(prop, "Bitrate", "Video bitrate (kb/s)");
 	RNA_def_property_update(prop, NC_SCENE|ND_RENDER_OPTIONS, NULL);