From 0de6cf5fdd9e1d89f5259eb0cc0795e424cfa6af Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 18 Jul 2015 17:55:19 +0200 Subject: avcodec/diracdec: Check for hpel_base allocation failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes null pointer dereference Fixes: signal_sigsegv_b02a96_280_RL_420p_ffdirac.drc with memlimit of 67108864 Found-by: Samuel Groß, Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit 1c5b712c0a643a039d6f34269b4102de313a050a) Signed-off-by: Michael Niedermayer --- libavcodec/diracdec.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c index b821d469e1..9640c82e83 100644 --- a/libavcodec/diracdec.c +++ b/libavcodec/diracdec.c @@ -1564,7 +1564,7 @@ static void select_dsp_funcs(DiracContext *s, int width, int height, int xblen, } } -static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, int width, int height) +static int interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, int width, int height) { /* chroma allocates an edge of 8 when subsampled which for 4:2:2 means an h edge of 16 and v edge of 8 @@ -1576,11 +1576,14 @@ static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, in /* no need for hpel if we only have fpel vectors */ if (!s->mv_precision) - return; + return 0; for (i = 1; i < 4; i++) { if (!ref->hpel_base[plane][i]) ref->hpel_base[plane][i] = av_malloc((height+2*edge) * ref->avframe->linesize[plane] + 32); + if (!ref->hpel_base[plane][i]) { + return AVERROR(ENOMEM); + } /* we need to be 16-byte aligned even for chroma */ ref->hpel[plane][i] = ref->hpel_base[plane][i] + edge*ref->avframe->linesize[plane] + 16; } @@ -1594,6 +1597,8 @@ static void interpolate_refplane(DiracContext *s, DiracFrame *ref, int plane, in s->mpvencdsp.draw_edges(ref->hpel[plane][3], ref->avframe->linesize[plane], width, height, edge, edge, EDGE_TOP | EDGE_BOTTOM); } ref->interpolated[plane] = 1; + + return 0; } /** @@ -1646,8 +1651,11 @@ static int dirac_decode_frame_internal(DiracContext *s) select_dsp_funcs(s, p->width, p->height, p->xblen, p->yblen); - for (i = 0; i < s->num_refs; i++) - interpolate_refplane(s, s->ref_pics[i], comp, p->width, p->height); + for (i = 0; i < s->num_refs; i++) { + int ret = interpolate_refplane(s, s->ref_pics[i], comp, p->width, p->height); + if (ret < 0) + return ret; + } memset(s->mctmp, 0, 4*p->yoffset*p->stride); -- cgit v1.2.3