diff options
author | John Brooks <john.brooks@bluecherry.net> | 2011-12-13 04:04:14 +0400 |
---|---|---|
committer | Anton Khirnov <anton@khirnov.net> | 2012-01-21 11:28:08 +0400 |
commit | 9b6aafba6c06ef62783dd5e9c5ed668f3a095128 (patch) | |
tree | 3a02ccf7a244c2326bdfe49bf1b5a853b2281cfb /libavcodec/mpegvideo_common.h | |
parent | 5d95112d926760046bc0b07aeda5cfc766ba54a6 (diff) |
mpegvideo: fix invalid memory access for small video dimensions
When either video dimension is only one macroblock, subtractions
based on v_edge_pos and the macroblock size may be negative. In
that situation, an unsigned comparison isn't sufficent to test for
MV overruns, because a limit of (unsigned)-1 will let any other
value pass.
Signed-off-by: Anton Khirnov <anton@khirnov.net>
Diffstat (limited to 'libavcodec/mpegvideo_common.h')
-rw-r--r-- | libavcodec/mpegvideo_common.h | 28 |
1 files changed, 14 insertions, 14 deletions
diff --git a/libavcodec/mpegvideo_common.h b/libavcodec/mpegvideo_common.h index d64404d8c5..9f6307ea7c 100644 --- a/libavcodec/mpegvideo_common.h +++ b/libavcodec/mpegvideo_common.h @@ -81,8 +81,8 @@ static inline void gmc1_motion(MpegEncContext *s, ptr = ref_picture[0] + (src_y * linesize) + src_x; if(s->flags&CODEC_FLAG_EMU_EDGE){ - if( (unsigned)src_x >= s->h_edge_pos - 17 - || (unsigned)src_y >= s->v_edge_pos - 17){ + if( (unsigned)src_x >= FFMAX(s->h_edge_pos - 17, 0) + || (unsigned)src_y >= FFMAX(s->v_edge_pos - 17, 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, linesize, 17, 17, src_x, src_y, s->h_edge_pos, s->v_edge_pos); ptr= s->edge_emu_buffer; } @@ -120,8 +120,8 @@ static inline void gmc1_motion(MpegEncContext *s, offset = (src_y * uvlinesize) + src_x; ptr = ref_picture[1] + offset; if(s->flags&CODEC_FLAG_EMU_EDGE){ - if( (unsigned)src_x >= (s->h_edge_pos>>1) - 9 - || (unsigned)src_y >= (s->v_edge_pos>>1) - 9){ + if( (unsigned)src_x >= FFMAX((s->h_edge_pos>>1) - 9, 0) + || (unsigned)src_y >= FFMAX((s->v_edge_pos>>1) - 9, 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, uvlinesize, 9, 9, src_x, src_y, s->h_edge_pos>>1, s->v_edge_pos>>1); ptr= s->edge_emu_buffer; emu=1; @@ -221,8 +221,8 @@ static inline int hpel_motion(MpegEncContext *s, src += src_y * stride + src_x; if(s->unrestricted_mv && (s->flags&CODEC_FLAG_EMU_EDGE)){ - if( (unsigned)src_x > h_edge_pos - (motion_x&1) - w - || (unsigned)src_y > v_edge_pos - (motion_y&1) - h){ + if( (unsigned)src_x > FFMAX(h_edge_pos - (motion_x&1) - w, 0) + || (unsigned)src_y > FFMAX(v_edge_pos - (motion_y&1) - h, 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, src, s->linesize, w+1, (h+1)<<field_based, src_x, src_y<<field_based, h_edge_pos, s->v_edge_pos); src= s->edge_emu_buffer; @@ -307,8 +307,8 @@ if(s->quarter_sample) ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x; ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x; - if( (unsigned)src_x > s->h_edge_pos - (motion_x&1) - 16 - || (unsigned)src_y > v_edge_pos - (motion_y&1) - h){ + if( (unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x&1) - 16, 0) + || (unsigned)src_y > FFMAX( v_edge_pos - (motion_y&1) - h , 0)){ if(is_mpeg12 || s->codec_id == CODEC_ID_MPEG2VIDEO || s->codec_id == CODEC_ID_MPEG1VIDEO){ av_log(s->avctx,AV_LOG_DEBUG, @@ -510,8 +510,8 @@ static inline void qpel_motion(MpegEncContext *s, ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x; ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x; - if( (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 16 - || (unsigned)src_y > v_edge_pos - (motion_y&3) - h ){ + if( (unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x&3) - 16, 0) + || (unsigned)src_y > FFMAX( v_edge_pos - (motion_y&3) - h , 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr_y, s->linesize, 17, 17+field_based, src_x, src_y<<field_based, s->h_edge_pos, s->v_edge_pos); @@ -588,8 +588,8 @@ static inline void chroma_4mv_motion(MpegEncContext *s, offset = src_y * s->uvlinesize + src_x; ptr = ref_picture[1] + offset; if(s->flags&CODEC_FLAG_EMU_EDGE){ - if( (unsigned)src_x > (s->h_edge_pos>>1) - (dxy &1) - 8 - || (unsigned)src_y > (s->v_edge_pos>>1) - (dxy>>1) - 8){ + if( (unsigned)src_x > FFMAX((s->h_edge_pos>>1) - (dxy &1) - 8, 0) + || (unsigned)src_y > FFMAX((s->v_edge_pos>>1) - (dxy>>1) - 8, 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, s->uvlinesize, 9, 9, src_x, src_y, s->h_edge_pos>>1, s->v_edge_pos>>1); @@ -760,8 +760,8 @@ static av_always_inline void MPV_motion_internal(MpegEncContext *s, ptr = ref_picture[0] + (src_y * s->linesize) + (src_x); if(s->flags&CODEC_FLAG_EMU_EDGE){ - if( (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 8 - || (unsigned)src_y > s->v_edge_pos - (motion_y&3) - 8 ){ + if( (unsigned)src_x > FFMAX(s->h_edge_pos - (motion_x&3) - 8, 0) + || (unsigned)src_y > FFMAX(s->v_edge_pos - (motion_y&3) - 8, 0)){ s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, s->linesize, 9, 9, src_x, src_y, |