avcodec/h264: Remove bits_per_raw_sample hacks

Fixes assertion failure

Fixes: f12c79769501ee471a5c731b8304ffcb/signal_sigabrt_7ffff6ac8cc9_3766_cov_4267830138_undemuxable_h264.ts

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 6 insertions, 9 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 18e5a4dd8b..05d238b468 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -619,7 +619,6 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
     h->bit_depth_luma    = 8;
     h->chroma_format_idc = 1;
 
-    h->avctx->bits_per_raw_sample = 8;
     h->cur_chroma_format_idc = 1;
 
     ff_h264dsp_init(&h->h264dsp, 8, 1);
@@ -654,6 +653,7 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
     ff_init_cabac_states();
 
     h->pixel_shift        = 0;
+    h->cur_bit_depth_luma =
     h->sps.bit_depth_luma = avctx->bits_per_raw_sample = 8;
 
     h->nb_slice_ctx = (avctx->active_thread_type & FF_THREAD_SLICE) ?  H264_MAX_THREADS : 1;
@@ -1237,7 +1237,7 @@ int ff_h264_set_parameter_from_sps(H264Context *h)
     if (h->avctx->has_b_frames < 2)
         h->avctx->has_b_frames = !h->low_delay;
 
-    if (h->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
+    if (h->cur_bit_depth_luma         != h->sps.bit_depth_luma ||
         h->cur_chroma_format_idc      != h->sps.chroma_format_idc) {
         if (h->avctx->codec &&
             h->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU &&
@@ -1248,6 +1248,7 @@ int ff_h264_set_parameter_from_sps(H264Context *h)
         }
         if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 14 &&
             h->sps.bit_depth_luma != 11 && h->sps.bit_depth_luma != 13) {
+            h->cur_bit_depth_luma         =
             h->avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
             h->cur_chroma_format_idc      = h->sps.chroma_format_idc;
             h->pixel_shift                = h->sps.bit_depth_luma > 8;
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 868d28512a..d27f0b4f7b 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -781,6 +781,7 @@ typedef struct H264Context {
     int initial_cpb_removal_delay[32];  ///< Initial timestamps for CPBs
 
     int cur_chroma_format_idc;
+    int cur_bit_depth_luma;
     int16_t slice_row[MAX_SLICES]; ///< to detect when MAX_SLICES is too low
 
     uint8_t parse_history[6];
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 44000c7288..1e28c604a9 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -458,11 +458,6 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
          h->sps.chroma_format_idc != h1->sps.chroma_format_idc ||
          h->sps.colorspace        != h1->sps.colorspace)) {
 
-        /* set bits_per_raw_sample to the previous value. the check for changed
-         * bit depth in h264_set_parameter_from_sps() uses it and sets it to
-         * the current value */
-        h->avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
-
         h->width     = h1->width;
         h->height    = h1->height;
         h->mb_height = h1->mb_height;
@@ -1303,7 +1298,7 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
 
         if (h->mb_width  != h->sps.mb_width ||
             h->mb_height != h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag) ||
-            h->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
+            h->cur_bit_depth_luma    != h->sps.bit_depth_luma ||
             h->cur_chroma_format_idc != h->sps.chroma_format_idc
         )
             needs_reinit = 1;
@@ -1325,7 +1320,7 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
     must_reinit = (h->context_initialized &&
                     (   16*h->sps.mb_width != h->avctx->coded_width
                      || 16*h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag) != h->avctx->coded_height
-                     || h->avctx->bits_per_raw_sample != h->sps.bit_depth_luma
+                     || h->cur_bit_depth_luma    != h->sps.bit_depth_luma
                      || h->cur_chroma_format_idc != h->sps.chroma_format_idc
                      || h->mb_width  != h->sps.mb_width
                      || h->mb_height != h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag)