vaapi_encode: Check packed header capabilities

This improves behaviour with drivers which do not support packed
headers, such as AMD VCE on mesa/gallium.

(cherry picked from commit 892bbbcdc171ff0d08d69636a240ffb95f54243c)

1 files changed, 15 insertions, 0 deletions

diff --git a/libavcodec/vaapi_encode_mjpeg.c b/libavcodec/vaapi_encode_mjpeg.c
index 3ca902a6af..3afce28a10 100644
--- a/libavcodec/vaapi_encode_mjpeg.c
+++ b/libavcodec/vaapi_encode_mjpeg.c
@@ -345,6 +345,17 @@ static av_cold int vaapi_encode_mjpeg_configure(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
 
+    // Hack: the implementation calls the JPEG image header (which we
+    // will use in the same way as a slice header) generic "raw data".
+    // Therefore, if after the packed header capability check we have
+    // PACKED_HEADER_RAW_DATA available, rewrite it as
+    // PACKED_HEADER_SLICE so that the header-writing code can do the
+    // right thing.
+    if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_RAW_DATA) {
+        ctx->va_packed_headers &= ~VA_ENC_PACKED_HEADER_RAW_DATA;
+        ctx->va_packed_headers |=  VA_ENC_PACKED_HEADER_SLICE;
+    }
+
     vaapi_encode_mjpeg_init_tables(avctx);
 
     return 0;
@@ -380,6 +391,10 @@ static av_cold int vaapi_encode_mjpeg_init(AVCodecContext *avctx)
 
     ctx->va_rc_mode = VA_RC_CQP;
 
+    // The JPEG image header - see note above.
+    ctx->va_packed_headers =
+        VA_ENC_PACKED_HEADER_RAW_DATA;
+
     ctx->surface_width  = FFALIGN(avctx->width,  8);
     ctx->surface_height = FFALIGN(avctx->height, 8);