apedec: check for filter buffer allocation failure

(cherry picked from commit 7500781313d11b37772c05a28da20fbc112db478)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 18 insertions, 13 deletions

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index f036c4a1d7..7cf72a00bd 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -163,6 +163,18 @@ typedef struct APEContext {
 
 // TODO: dsputilize
 
+static av_cold int ape_decode_close(AVCodecContext * avctx)
+{
+    APEContext *s = avctx->priv_data;
+    int i;
+
+    for (i = 0; i < APE_FILTER_LEVELS; i++)
+        av_freep(&s->filterbuf[i]);
+
+    av_freep(&s->data);
+    return 0;
+}
+
 static av_cold int ape_decode_init(AVCodecContext * avctx)
 {
     APEContext *s = avctx->priv_data;
@@ -195,25 +207,18 @@ static av_cold int ape_decode_init(AVCodecContext * avctx)
     for (i = 0; i < APE_FILTER_LEVELS; i++) {
         if (!ape_filter_orders[s->fset][i])
             break;
-        s->filterbuf[i] = av_malloc((ape_filter_orders[s->fset][i] * 3 + HISTORY_SIZE) * 4);
+        FF_ALLOC_OR_GOTO(avctx, s->filterbuf[i],
+                         (ape_filter_orders[s->fset][i] * 3 + HISTORY_SIZE) * 4,
+                         filter_alloc_fail);
     }
 
     dsputil_init(&s->dsp, avctx);
     avctx->sample_fmt = AV_SAMPLE_FMT_S16;
     avctx->channel_layout = (avctx->channels==2) ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO;
     return 0;
-}
-
-static av_cold int ape_decode_close(AVCodecContext * avctx)
-{
-    APEContext *s = avctx->priv_data;
-    int i;
-
-    for (i = 0; i < APE_FILTER_LEVELS; i++)
-        av_freep(&s->filterbuf[i]);
-
-    av_freep(&s->data);
-    return 0;
+filter_alloc_fail:
+    ape_decode_close(avctx);
+    return AVERROR(ENOMEM);
 }
 
 /**