diff options
author | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-01 23:56:05 +0400 |
---|---|---|
committer | Ronald S. Bultje <rsbultje@gmail.com> | 2012-03-02 00:15:51 +0400 |
commit | 0ce4fe482c27abfa7eac503a52fdc50b70ccd871 (patch) | |
tree | 76d88c27d41c95262b514b84240351514a52e54c /libavcodec/h264.c | |
parent | cc412b71047ebf77c7e810c90b044f018a1c0c2d (diff) |
h264: error out on invalid bitdepth.
Fixes invalid reads while initializing the dequant tables, which uses
the bit depth to determine the QP table size.
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Diffstat (limited to 'libavcodec/h264.c')
-rw-r--r-- | libavcodec/h264.c | 17 |
1 files changed, 11 insertions, 6 deletions
diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 6eb24560b8..de79dba3af 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2707,11 +2707,6 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ s->avctx->level = h->sps.level_idc; s->avctx->refs = h->sps.ref_frame_count; - if(h == h0 && h->dequant_coeff_pps != pps_id){ - h->dequant_coeff_pps = pps_id; - init_dequant_tables(h); - } - s->mb_width= h->sps.mb_width; s->mb_height= h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag); @@ -2786,7 +2781,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ else s->avctx->pix_fmt = PIX_FMT_YUV420P10; break; - default: + case 8: if (CHROMA444){ if (s->avctx->colorspace == AVCOL_SPC_RGB) { s->avctx->pix_fmt = PIX_FMT_GBRP; @@ -2802,6 +2797,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ hwaccel_pixfmt_list_h264_jpeg_420 : ff_hwaccel_pixfmt_list_420); } + break; + default: + av_log(s->avctx, AV_LOG_ERROR, + "Unsupported bit depth: %d\n", h->sps.bit_depth_luma); + return AVERROR_INVALIDDATA; } s->avctx->hwaccel = ff_find_hwaccel(s->avctx->codec->id, s->avctx->pix_fmt); @@ -2846,6 +2846,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ } } + if(h == h0 && h->dequant_coeff_pps != pps_id){ + h->dequant_coeff_pps = pps_id; + init_dequant_tables(h); + } + h->frame_num= get_bits(&s->gb, h->sps.log2_max_frame_num); h->mb_mbaff = 0; |