dxva2: factor codec checks into dxva_common

3 files changed, 42 insertions, 14 deletions

diff --git a/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp b/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp
index 9d454df4..f51685ac 100644
--- a/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp
+++ b/decoder/LAVVideo/decoders/dxva2/dxva_common.cpp
@@ -154,3 +154,32 @@ int check_dxva_mode_compatibility(const dxva_mode_t *mode, int codec, int profil
 
   return 1;
 }
+
+int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int hwpixfmt)
+{
+  // check mpeg2 pixfmt
+  if (codec == AV_CODEC_ID_MPEG2VIDEO && pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUVJ420P && pix_fmt != hwpixfmt && pix_fmt != AV_PIX_FMT_NONE)
+    return 1;
+
+  // check h264 pixfmt
+  if (codec == AV_CODEC_ID_H264 && pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUVJ420P && pix_fmt != hwpixfmt && pix_fmt != AV_PIX_FMT_NONE)
+    return 1;
+
+  // check h264 profile
+  if (codec == AV_CODEC_ID_H264 && profile != FF_PROFILE_UNKNOWN && !H264_CHECK_PROFILE(profile))
+    return 1;
+
+  // check wmv/vc1 profile
+  if ((codec == AV_CODEC_ID_WMV3 || codec == AV_CODEC_ID_VC1) && profile == FF_PROFILE_VC1_COMPLEX)
+    return 1;
+
+  // check hevc profile/pixfmt
+  if (codec == AV_CODEC_ID_HEVC && (!HEVC_CHECK_PROFILE(profile) || (pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUVJ420P && pix_fmt != AV_PIX_FMT_YUV420P10 && pix_fmt != hwpixfmt && pix_fmt != AV_PIX_FMT_NONE)))
+    return 1;
+
+  // check vp9 profile/pixfmt
+  if (codec == AV_CODEC_ID_VP9 && (!VP9_CHECK_PROFILE(profile) || (pix_fmt != AV_PIX_FMT_YUV420P && pix_fmt != AV_PIX_FMT_YUV420P10 && pix_fmt != AV_PIX_FMT_DXVA2_VLD && pix_fmt != AV_PIX_FMT_NONE)))
+    return 1;
+
+  return 0;
+}
diff --git a/decoder/LAVVideo/decoders/dxva2/dxva_common.h b/decoder/LAVVideo/decoders/dxva2/dxva_common.h
index f9e1b94f..fe74a5ff 100644
--- a/decoder/LAVVideo/decoders/dxva2/dxva_common.h
+++ b/decoder/LAVVideo/decoders/dxva2/dxva_common.h
@@ -35,3 +35,14 @@ extern const dxva_mode_t dxva_modes[];
 
 const dxva_mode_t *get_dxva_mode_from_guid(const GUID *guid);
 int check_dxva_mode_compatibility(const dxva_mode_t *mode, int codec, int profile);
+
+int check_dxva_codec_profile(AVCodecID codec, AVPixelFormat pix_fmt, int profile, int hwpixfmt);
+
+#define H264_CHECK_PROFILE(profile) \
+  (((profile) & ~FF_PROFILE_H264_CONSTRAINED) <= FF_PROFILE_H264_HIGH)
+
+#define HEVC_CHECK_PROFILE(profile) \
+  ((profile) <= FF_PROFILE_HEVC_MAIN_10)
+
+#define VP9_CHECK_PROFILE(profile) \
+  ((profile) == FF_PROFILE_VP9_0 || (profile) == FF_PROFILE_VP9_2)
diff --git a/decoder/LAVVideo/decoders/dxva2dec.cpp b/decoder/LAVVideo/decoders/dxva2dec.cpp
index 3609a0a7..60e64611 100644
--- a/decoder/LAVVideo/decoders/dxva2dec.cpp
+++ b/decoder/LAVVideo/decoders/dxva2dec.cpp
@@ -884,15 +884,6 @@ STDMETHODIMP CDecDXVA2::Init()
   return S_OK;
 }
 
-#define H264_CHECK_PROFILE(profile) \
-  (((profile) & ~FF_PROFILE_H264_CONSTRAINED) <= FF_PROFILE_H264_HIGH)
-
-#define HEVC_CHECK_PROFILE(profile) \
-  ((profile) <= FF_PROFILE_HEVC_MAIN_10)
-
-#define VP9_CHECK_PROFILE(profile) \
-  ((profile) == FF_PROFILE_VP9_0 || (profile) == FF_PROFILE_VP9_2)
-
 STDMETHODIMP CDecDXVA2::InitDecoder(AVCodecID codec, const CMediaType *pmt)
 {
   HRESULT hr = S_OK;
@@ -959,11 +950,8 @@ STDMETHODIMP CDecDXVA2::InitDecoder(AVCodecID codec, const CMediaType *pmt)
       output = (D3DFORMAT)FOURCC_NV12;
   }
 
-  if (((codec == AV_CODEC_ID_H264 || codec == AV_CODEC_ID_MPEG2VIDEO) && m_pAVCtx->pix_fmt != AV_PIX_FMT_YUV420P && m_pAVCtx->pix_fmt != AV_PIX_FMT_YUVJ420P && m_pAVCtx->pix_fmt != AV_PIX_FMT_DXVA2_VLD && m_pAVCtx->pix_fmt != AV_PIX_FMT_NONE)
-    || (codec == AV_CODEC_ID_H264 && m_pAVCtx->profile != FF_PROFILE_UNKNOWN && !H264_CHECK_PROFILE(m_pAVCtx->profile))
-    || ((codec == AV_CODEC_ID_WMV3 || codec == AV_CODEC_ID_VC1) && m_pAVCtx->profile == FF_PROFILE_VC1_COMPLEX)
-    || (codec == AV_CODEC_ID_HEVC && (!HEVC_CHECK_PROFILE(m_pAVCtx->profile) || (m_pAVCtx->pix_fmt != AV_PIX_FMT_YUV420P && m_pAVCtx->pix_fmt != AV_PIX_FMT_YUVJ420P && m_pAVCtx->pix_fmt != AV_PIX_FMT_YUV420P10 && m_pAVCtx->pix_fmt != AV_PIX_FMT_DXVA2_VLD && m_pAVCtx->pix_fmt != AV_PIX_FMT_NONE)))
-    || (codec == AV_CODEC_ID_VP9 && (!VP9_CHECK_PROFILE(m_pAVCtx->profile) || (m_pAVCtx->pix_fmt != AV_PIX_FMT_YUV420P && m_pAVCtx->pix_fmt != AV_PIX_FMT_YUV420P10 && m_pAVCtx->pix_fmt != AV_PIX_FMT_DXVA2_VLD && m_pAVCtx->pix_fmt != AV_PIX_FMT_NONE)))) {
+  if (check_dxva_codec_profile(m_pAVCtx->codec_id, m_pAVCtx->pix_fmt, m_pAVCtx->profile, AV_PIX_FMT_DXVA2_VLD))
+  {
     DbgLog((LOG_TRACE, 10, L"-> Incompatible profile detected, falling back to software decoding"));
     return E_FAIL;
   }