avfilter/all: propagate errors of functions from avfilter/formats

Many of the functions from avfilter/formats can return errors, usually AVERROR(ENOMEM).
This propagates the return values.

All of these were found by using av_warn_unused_result, demonstrating its utility.

Tested with FATE. I am least sure of the changes to avfilter/filtergraph,
since I don't know what/how reduce_format is intended to behave and how it should
react to errors.

Fixes: CID 1325680, 1325679, 1325678.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Previous version Reviewed-by: Nicolas George <george@nsup.org>
Previous version Reviewed-by: Clément Bœsch <u@pkh.me>
Signed-off-by: Ganesh Ajjanagadde <gajjanagadde@gmail.com>

1 files changed, 17 insertions, 11 deletions

diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
index 81c9cc1665..3c61731431 100644
--- a/libavfilter/vf_overlay.c
+++ b/libavfilter/vf_overlay.c
@@ -247,31 +247,37 @@ static int query_formats(AVFilterContext *ctx)
 
     AVFilterFormats *main_formats;
     AVFilterFormats *overlay_formats;
+    int ret;
 
     switch (s->format) {
     case OVERLAY_FORMAT_YUV420:
-        main_formats    = ff_make_format_list(main_pix_fmts_yuv420);
-        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420);
+        if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv420)) ||
+            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420)))
+            return AVERROR(ENOMEM);
         break;
     case OVERLAY_FORMAT_YUV422:
-        main_formats    = ff_make_format_list(main_pix_fmts_yuv422);
-        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422);
+        if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv422)) ||
+            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422)))
+            return AVERROR(ENOMEM);
         break;
     case OVERLAY_FORMAT_YUV444:
-        main_formats    = ff_make_format_list(main_pix_fmts_yuv444);
-        overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444);
+        if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv444)) ||
+            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444)))
+            return AVERROR(ENOMEM);
         break;
     case OVERLAY_FORMAT_RGB:
-        main_formats    = ff_make_format_list(main_pix_fmts_rgb);
-        overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb);
+        if (!(main_formats    = ff_make_format_list(main_pix_fmts_rgb)) ||
+            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb)))
+            return AVERROR(ENOMEM);
         break;
     default:
         av_assert0(0);
     }
 
-    ff_formats_ref(main_formats,    &ctx->inputs [MAIN   ]->out_formats);
-    ff_formats_ref(overlay_formats, &ctx->inputs [OVERLAY]->out_formats);
-    ff_formats_ref(main_formats,    &ctx->outputs[MAIN   ]->in_formats );
+    if ((ret = ff_formats_ref(main_formats   , &ctx->inputs[MAIN]->out_formats   )) < 0 ||
+        (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
+        (ret = ff_formats_ref(main_formats   , &ctx->outputs[MAIN]->in_formats   )) < 0)
+        return ret;
 
     return 0;
 }