Patch #34204: [Render Animation] Fails with "Error: Specified sample_fmt is not supported" with ogg vorbis

Patch by Jehan Pages (pardon for mis-typing, emacs-nox works not so good with
urf-8 buffers here), with some own modifications. Thanks!

From the patch tracker:

The problem is because of several versions of ffmpeg, but even more because of
the fork situation libav/ffmpeg. So there are some installed versions out there
where you *must* use a float sample for some codec; whereas oppositely on some
other installations, you *must* use the int sample. So for some people, one
works not the other, and reciprocally.

As a consequence, you can't just have a switch codec-based, like in current
code, which decides on the float or int implementation, you must necessarily
have a runtime test because you won't know until then if ogg vorbis will use
one or another sample (note: that's true also for AC3 as I fixed the exact same
bug in DVDStyler for AC3 encoding a few months ago; and I guess it would be same
for AAC).

Some notes from self:
- New FFmpeg requires using FLTP for AAC, AC3 and Vorbis, it's not supported
  by audaspace and result in this case would be just wrong. Throw an error
  in cases FLTP is trying to be used.
- Moved strict_std_compliance a bit upper. When we'll support FLTP both
  FLT and FLTP for AAC would need to be using FF_COMPLIANCE_EXPERIMENTAL.
- It is nice to have such check of supported by codec formats anyway.

1 files changed, 44 insertions, 4 deletions

diff --git a/source/blender/blenkernel/intern/writeffmpeg.c b/source/blender/blenkernel/intern/writeffmpeg.c
index af8b32ab75c..082ae380070 100644
--- a/source/blender/blenkernel/intern/writeffmpeg.c
+++ b/source/blender/blenkernel/intern/writeffmpeg.c
@@ -111,8 +111,9 @@ static void delete_picture(AVFrame *f)
 	}
 }
 
-static int use_float_audio_buffer(int codec_id)
+static int request_float_audio_buffer(int codec_id)
 {
+	/* If any of these codecs, we prefer the float sample format (if supported) */
 	return codec_id == CODEC_ID_AAC || codec_id == CODEC_ID_AC3 || codec_id == CODEC_ID_VORBIS;
 }
 
@@ -629,16 +630,55 @@ static AVStream *alloc_audio_stream(RenderData *rd, int codec_id, AVFormatContex
 	c->bit_rate = ffmpeg_audio_bitrate * 1000;
 	c->sample_fmt = AV_SAMPLE_FMT_S16;
 	c->channels = rd->ffcodecdata.audio_channels;
-	if (use_float_audio_buffer(codec_id)) {
+
+	if (request_float_audio_buffer(codec_id)) {
+		/* mainly for AAC codec which is experimental */
 		c->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
 		c->sample_fmt = AV_SAMPLE_FMT_FLT;
 	}
+
 	codec = avcodec_find_encoder(c->codec_id);
 	if (!codec) {
 		//XXX error("Couldn't find a valid audio codec");
 		return NULL;
 	}
 
+	if (codec->sample_fmts) {
+		/* check if the prefered sample format for this codec is supported.
+		 * this is because, depending on the version of libav, and with the whole ffmpeg/libav fork situation,
+		 * you have various implementations around. float samples in particular are not always supported.
+		 */
+		const enum AVSampleFormat *p = codec->sample_fmts;
+		for (; *p!=-1; p++) {
+			if (*p == st->codec->sample_fmt)
+				break;
+		}
+		if (*p == -1) {
+			/* sample format incompatible with codec. Defaulting to a format known to work */
+			st->codec->sample_fmt = codec->sample_fmts[0];
+		}
+	}
+
+	if (c->sample_fmt == AV_SAMPLE_FMT_FLTP) {
+		BLI_strncpy(error, "Requested audio codec requires planar float sample format, which is not supported yet", error_size);
+		return NULL;
+	}
+
+	if (codec->supported_samplerates) {
+		const int *p = codec->supported_samplerates;
+		int best = 0;
+		int best_dist = INT_MAX;
+		for (; *p; p++){
+			int dist = abs(st->codec->sample_rate - *p);
+			if (dist < best_dist){
+				best_dist = dist;
+				best = *p;
+			}
+		}
+		/* best is the closest supported sample rate (same as selected if best_dist == 0) */
+		st->codec->sample_rate = best;
+	}
+
 	set_ffmpeg_properties(rd, c, "audio");
 
 	if (avcodec_open2(c, codec, NULL) < 0) {
@@ -664,7 +704,7 @@ static AVStream *alloc_audio_stream(RenderData *rd, int codec_id, AVFormatContex
 
 	audio_output_buffer = (uint8_t *) av_malloc(audio_outbuf_size);
 
-	if (use_float_audio_buffer(codec_id)) {
+	if (c->sample_fmt == AV_SAMPLE_FMT_FLT) {
 		audio_input_buffer = (uint8_t *) av_malloc(audio_input_samples * c->channels * sizeof(float));
 	}
 	else {
@@ -970,7 +1010,7 @@ int BKE_ffmpeg_start(struct Scene *scene, RenderData *rd, int rectx, int recty,
 		AVCodecContext *c = audio_stream->codec;
 		AUD_DeviceSpecs specs;
 		specs.channels = c->channels;
-		if (use_float_audio_buffer(c->codec_id)) {
+		if (c->sample_fmt == AV_SAMPLE_FMT_FLT) {
 			specs.format = AUD_FORMAT_FLOAT32;
 		}
 		else {