diff options
author | Anton Khirnov <anton@khirnov.net> | 2013-11-28 13:54:35 +0400 |
---|---|---|
committer | Anton Khirnov <anton@khirnov.net> | 2014-01-06 11:25:36 +0400 |
commit | a03a642d5ceb5f2f7c6ebbf56ff365dfbcdb65eb (patch) | |
tree | c030c23ab383f313b633dcea9ef20d89ae4668d9 /libavcodec/x86 | |
parent | 9eef9eb3014b2ed9c3ff4aac510a9f04edb555cf (diff) |
h264: do not use 422 functions for monochrome
Fixes invalid memory access.
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org
Diffstat (limited to 'libavcodec/x86')
-rw-r--r-- | libavcodec/x86/h264_intrapred_init.c | 20 | ||||
-rw-r--r-- | libavcodec/x86/h264dsp_init.c | 14 |
2 files changed, 17 insertions, 17 deletions
diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c index 6dd98aab72..f934256706 100644 --- a/libavcodec/x86/h264_intrapred_init.c +++ b/libavcodec/x86/h264_intrapred_init.c @@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_MMX(cpu_flags)) { h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vertical_8_mmx; h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmx; - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8 [VERT_PRED8x8 ] = ff_pred8x8_vertical_8_mmx; h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmx; } @@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_mmx; h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmx; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx; if (codec_id == AV_CODEC_ID_SVQ3) { if (cpu_flags & AV_CPU_FLAG_CMOV) @@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_MMXEXT(cpu_flags)) { h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmxext; h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_mmxext; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmxext; h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_mmxext; h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_mmxext; @@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4 [HOR_UP_PRED ] = ff_pred4x4_horizontal_up_8_mmxext; } if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) { - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_8_mmxext; h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_8_mmxext; } @@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmxext; h->pred4x4 [VERT_PRED ] = ff_pred4x4_vertical_vp8_8_mmxext; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_svq3_8_mmxext; @@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_tm_vp8_8_sse2; h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_sse2; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2; @@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_SSSE3(cpu_flags)) { h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_ssse3; h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_ssse3; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_ssse3; h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_ssse3; h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_ssse3; @@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_ssse3; h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_ssse3; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3; @@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4[DC_PRED ] = ff_pred4x4_dc_10_mmxext; h->pred4x4[HOR_UP_PRED ] = ff_pred4x4_horizontal_up_10_mmxext; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_mmxext; h->pred8x8l[DC_128_PRED ] = ff_pred8x8l_128_dc_10_mmxext; @@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4[VERT_RIGHT_PRED ] = ff_pred4x4_vertical_right_10_sse2; h->pred4x4[HOR_DOWN_PRED ] = ff_pred4x4_horizontal_down_10_sse2; - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_sse2; h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_10_sse2; h->pred8x8[PLANE_PRED8x8 ] = ff_pred8x8_plane_10_sse2; diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c index 4164b833db..e9d93e0af9 100644 --- a/libavcodec/x86/h264dsp_init.c +++ b/libavcodec/x86/h264dsp_init.c @@ -213,7 +213,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, { int cpu_flags = av_get_cpu_flags(); - if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(cpu_flags)) + if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(cpu_flags)) c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext; if (bit_depth == 8) { @@ -225,7 +225,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->h264_idct_add16 = ff_h264_idct_add16_8_mmx; c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) c->h264_idct_add8 = ff_h264_idct_add8_8_mmx; c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx; if (cpu_flags & AV_CPU_FLAG_CMOV) @@ -236,13 +236,13 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext; c->h264_idct_add16 = ff_h264_idct_add16_8_mmxext; c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmxext; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext; c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext; c->h264_v_loop_filter_chroma = ff_deblock_v_chroma_8_mmxext; c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext; - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { c->h264_h_loop_filter_chroma = ff_deblock_h_chroma_8_mmxext; c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext; } @@ -265,7 +265,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->h264_idct_add16 = ff_h264_idct_add16_8_sse2; c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) c->h264_idct_add8 = ff_h264_idct_add8_8_sse2; c->h264_idct_add16intra = ff_h264_idct_add16intra_8_sse2; c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2; @@ -308,7 +308,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2; c->h264_idct_add16 = ff_h264_idct_add16_10_sse2; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) c->h264_idct_add8 = ff_h264_idct_add8_10_sse2; c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2; #if HAVE_ALIGNED_STACK @@ -348,7 +348,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx; c->h264_idct_add16 = ff_h264_idct_add16_10_avx; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) c->h264_idct_add8 = ff_h264_idct_add8_10_avx; c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx; #if HAVE_ALIGNED_STACK |