Fix overflow in 10->8 bit dithering

3 files changed, 5 insertions, 4 deletions

diff --git a/decoder/LAVVideo/pixconv/pixconv_sse2_templates.h b/decoder/LAVVideo/pixconv/pixconv_sse2_templates.h
index e02546ea..47d653c7 100644
--- a/decoder/LAVVideo/pixconv/pixconv_sse2_templates.h
+++ b/decoder/LAVVideo/pixconv/pixconv_sse2_templates.h
@@ -36,8 +36,9 @@
 // shift - shift offset to 8-bit (ie. 2 for 10bit)
 #define PIXCONV_LOAD_PIXEL16_DITHER(reg,dreg,src,shift)               \
   reg = _mm_load_si128((const __m128i *)(src)); /* load (aligned) */  \
+  reg = _mm_slli_epi16(reg, 8-shift);           /* shift to 16-bit */ \
   reg = _mm_adds_epu16(reg, dreg);              /* dither */          \
-  reg = _mm_srli_epi16(reg, shift);             /* shift to 8-bit */
+  reg = _mm_srli_epi16(reg, 8);                 /* shift to 8-bit */
 
 // Load 8 16-bit pixels into a register, and dither them to 8 bit
 // The 8-bit pixels will be in the 8 low-bytes in the register
diff --git a/decoder/LAVVideo/pixconv/yuv2yuv_unscaled.cpp b/decoder/LAVVideo/pixconv/yuv2yuv_unscaled.cpp
index 40425e3c..d889a661 100644
--- a/decoder/LAVVideo/pixconv/yuv2yuv_unscaled.cpp
+++ b/decoder/LAVVideo/pixconv/yuv2yuv_unscaled.cpp
@@ -49,7 +49,7 @@ DECLARE_CONV_FUNC_IMPL(convert_yuv420_yv12_nv12_dither_le)
   // Process Y
   for (line = 0; line < height; ++line) {
     // Load dithering coefficients for this line
-    PIXCONV_LOAD_DITHER_COEFFS(xmm4,line,shift,dithers);
+    PIXCONV_LOAD_DITHER_COEFFS(xmm4,line,8,dithers);
 
     __m128i *dst128Y = (__m128i *)(dst + line * outYStride);
 
@@ -70,7 +70,7 @@ DECLARE_CONV_FUNC_IMPL(convert_yuv420_yv12_nv12_dither_le)
 
   for (line = 0; line < (height >> 1); ++line) {
     // Load dithering coefficients for this line
-    PIXCONV_LOAD_DITHER_COEFFS(xmm4,line,shift,dithers);
+    PIXCONV_LOAD_DITHER_COEFFS(xmm4,line,8,dithers);
 
     __m128i *dst128UV = (__m128i *)(dstV + line * outYStride);
     __m128i *dst128U = (__m128i *)(dstU + line * outUVStride);
diff --git a/decoder/LAVVideo/pixconv/yuv444_ayuv.cpp b/decoder/LAVVideo/pixconv/yuv444_ayuv.cpp
index 1eb15a46..3869bb27 100644
--- a/decoder/LAVVideo/pixconv/yuv444_ayuv.cpp
+++ b/decoder/LAVVideo/pixconv/yuv444_ayuv.cpp
@@ -107,7 +107,7 @@ DECLARE_CONV_FUNC_IMPL(convert_yuv444_ayuv_dither_le)
 
   for (line = 0; line < height; ++line) {
     // Load dithering coefficients for this line
-    PIXCONV_LOAD_DITHER_COEFFS(xmm7,line,shift,dithers);
+    PIXCONV_LOAD_DITHER_COEFFS(xmm7,line,8,dithers);
 
     __m128i *dst128 = (__m128i *)(dst + line * outStride);