diff options
author | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-05 17:27:52 +0400 |
---|---|---|
committer | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-22 21:35:14 +0400 |
commit | f99654d47030daca949cc6bfb14aa8e8455e8e59 (patch) | |
tree | 27da15556f9acf5f6a52233f35e3db5c0617de33 /libswscale | |
parent | ff7913aef14b340320b16900d4127b62ead776c9 (diff) |
swscale: reintroduce full precision in 16-bit output.
Diffstat (limited to 'libswscale')
-rw-r--r-- | libswscale/swscale.c | 16 |
1 files changed, 11 insertions, 5 deletions
diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 1d9b965ac9..8ed68c5980 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -202,20 +202,26 @@ yuv2yuvX16_c_template(const int16_t *filter, int filterSize, { #define output_pixel(pos, val) \ if (big_endian) { \ - AV_WB16(pos, av_clip_uint16(val >> shift)); \ + AV_WB16(pos, 0x8000 + av_clip_int16(val >> shift)); \ } else { \ - AV_WL16(pos, av_clip_uint16(val >> shift)); \ + AV_WL16(pos, 0x8000 + av_clip_int16(val >> shift)); \ } int i; - int shift = 15 + 16 - output_bits - 1; + int shift = 15 + 16 - output_bits; for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits - 1); + int val = 1 << (30-output_bits); int j; + /* range of val is [0,0x7FFFFFFF], so 31 bits, but with lanczos/spline + * filters (or anything with negative coeffs, the range can be slightly + * wider in both directions. To account for this overflow, we subtract + * a constant so it always fits in the signed range (assuming a + * reasonable filterSize), and re-add that at the end. */ + val -= 0x40000000; for (j = 0; j < filterSize; j++) - val += (src[j][i] * filter[j]) >> 1; + val += src[j][i] * filter[j]; output_pixel(&dest[i], val); } |