h264: fix ff_generate_sliding_window_mmcos() prototype.

It's been returning an error value since
bad446e251405dc250c3cbee199072e083a1e4b9

Also check for the errors it returns.
(cherry picked from commit ea382767ad2191acbe97e90624059723e15f0e4b)

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

3 files changed, 12 insertions, 6 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index e8b3e723e1..ebf8d0b2ea 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2904,7 +2904,9 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             s->current_picture_ptr->frame_num = h->prev_frame_num;
             ff_thread_report_progress(&s->current_picture_ptr->f, INT_MAX, 0);
             ff_thread_report_progress(&s->current_picture_ptr->f, INT_MAX, 1);
-            ff_generate_sliding_window_mmcos(h, 1);
+            if ((ret = ff_generate_sliding_window_mmcos(h, 1)) < 0 &&
+                s->avctx->err_recognition & AV_EF_EXPLODE)
+                return ret;
             if (ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index) < 0 &&
                 (s->avctx->err_recognition & AV_EF_EXPLODE))
                 return AVERROR_INVALIDDATA;
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index ad4732e1df..898ebf7b0a 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -648,7 +648,7 @@ int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count);
 int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
                                    int first_slice);
 
-void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice);
+int ff_generate_sliding_window_mmcos(H264Context *h, int first_slice);
 
 /**
  * Check if the top & left blocks are available if needed & change the
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 62c2a5758e..0e4bd76931 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -488,7 +488,7 @@ static int check_opcodes(MMCO *mmco1, MMCO *mmco2, int n_mmcos)
     return 0;
 }
 
-void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
+int ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
 {
     MpegEncContext * const s = &h->s;
     MMCO mmco_temp[MAX_MMCO_COUNT], *mmco = first_slice ? h->mmco : mmco_temp;
@@ -521,6 +521,7 @@ void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
                mmco_index, h->mmco_index, i);
         return AVERROR_INVALIDDATA;
     }
+    return 0;
 }
 
 int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count){
@@ -686,7 +687,7 @@ int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
                                    int first_slice)
 {
     MpegEncContext * const s = &h->s;
-    int i;
+    int i, ret;
     MMCO mmco_temp[MAX_MMCO_COUNT], *mmco = first_slice ? h->mmco : mmco_temp;
     int mmco_index = 0;
 
@@ -743,8 +744,11 @@ int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
             }
             mmco_index = i;
         } else {
-            if (first_slice)
-                ff_generate_sliding_window_mmcos(h, first_slice);
+            if (first_slice) {
+                ret = ff_generate_sliding_window_mmcos(h, first_slice);
+                if (ret < 0 && s->avctx->err_recognition & AV_EF_EXPLODE)
+                    return ret;
+            }
             mmco_index = -1;
         }
     }