libswcale/input: use more accurate planer rgb16 yuv conversions

These conversion appears to be exhibiting the same rounding error as the rgbf32 formats where.
I seperated the rounding value from the 16 and 128 offsets, I think it makes it a little more clear.

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

12 files changed, 50 insertions, 50 deletions

diff --git a/tests/ref/fate/filter-pixfmts-scale b/tests/ref/fate/filter-pixfmts-scale
index 30e7cd5b06..f47c9b887f 100644
--- a/tests/ref/fate/filter-pixfmts-scale
+++ b/tests/ref/fate/filter-pixfmts-scale
@@ -19,25 +19,25 @@ bgra                d8316272bc3a360ef9dff3ecc84520a3
 bgra64be            4e6a1b9f9c18b881c27d76611d45f737
 bgra64le            efeee0abcc658ebcff049d5e74d74943
 gbrap               4a100f750ac846b34bfeef0d6893c3de
-gbrap10be           6d89abb9248006c3e9017545e9474654
-gbrap10le           cf974e23f485a10740f5de74a5c8c3df
-gbrap12be           1d9b57766ba9c2192403f43967cb9af0
-gbrap12le           bb1ba1c157717db3dd612a76d38a018e
-gbrap16be           c72b935a6e57a8e1c37bff08c2db55b1
-gbrap16le           13eb0e62b1ac9c1c86c81521eaefab5f
+gbrap10be           dc6aea3559ea4fcdda1ccc4f23d2f2fb
+gbrap10le           6e1cba57029fdf0f9d46b5e5cd55112b
+gbrap12be           dbe3a662c016563529032cd4dfb80262
+gbrap12le           24f5ecb32435b73353517e017c165e31
+gbrap16be           31968e6872a46e8174fb57f8920ed10d
+gbrap16le           8c6758f33671b673b6d30969fc05a23d
 gbrapf32be          366b804d5697276e8c481c4bdf05a00b
 gbrapf32le          558a268e6d6b907449d1056afab78f29
 gbrp                dc3387f925f972c61aae7eb23cdc19f0
-gbrp10be            0277d4c3a8498d75e2783fb81379e481
-gbrp10le            f3d70f8ab845c3c9b8f7452e4a6e285a
-gbrp12be            fbd4e149c452c351c6d1c11d6b6e176a
-gbrp12le            c51d51c3b753d735eb22983397262c88
-gbrp14be            cd20808592e62cc439786c18a14b3e70
-gbrp14le            456f7d1ff5990aa6379137d84dd63862
-gbrp16be            5fc826cfabebfc1442cb793c4b6303e2
-gbrp16le            1b3e0b63d47a3e1b6b20931316883bf2
-gbrp9be             d9c88968001e1452ff31fbc8d16b18a0
-gbrp9le             2ccfed0816bf6bd4bb3a5b7591d9603a
+gbrp10be            a318ea42e53a7b80a55aa7c19c9a0ab5
+gbrp10le            994e8fc6a1e5b230f4c55893fd7618d6
+gbrp12be            bfbd419dd18c0a5677d4bce55ab24e2e
+gbrp12le            95af1f8495b2a7a7ad67802e3e8bca01
+gbrp14be            9d8113c9a5182c02dbe4576509f59a96
+gbrp14le            952f39881e500ed684c8b216185f4b80
+gbrp16be            5241eee3465096efa111b86b30c3aaaa
+gbrp16le            5b8b997378ce31207f37059dbfb40c4a
+gbrp9be             d7caf58cc3a74a036e11f924f03fc04c
+gbrp9le             010f7bcd8b2e17065d01a09f0d483218
 gbrpf32be           f3d0cefdf11c861001880772d817aac8
 gbrpf32le           290468205c1c18a0667edfca45061aee
 gray                221201cc7cfc4964eacd8b3e426fd276
diff --git a/tests/ref/fate/psd-rgb48 b/tests/ref/fate/psd-rgb48
index f60cbab078..139b3c1df3 100644
--- a/tests/ref/fate/psd-rgb48
+++ b/tests/ref/fate/psd-rgb48
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 128x128
 #sar 0: 0/1
-0,          0,          0,        1,    49152, 0x0bc6503a
+0,          0,          0,        1,    49152, 0x9fb6903c
diff --git a/tests/ref/fate/psd-rgba64 b/tests/ref/fate/psd-rgba64
index e51686767e..ed36c339d1 100644
--- a/tests/ref/fate/psd-rgba64
+++ b/tests/ref/fate/psd-rgba64
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 128x128
 #sar 0: 0/1
-0,          0,          0,        1,    49152, 0xb847fd73
+0,          0,          0,        1,    49152, 0x05856a20
diff --git a/tests/ref/fate/sws-floatimg-cmp b/tests/ref/fate/sws-floatimg-cmp
index cf6788fc23..251042f1c3 100644
--- a/tests/ref/fate/sws-floatimg-cmp
+++ b/tests/ref/fate/sws-floatimg-cmp
@@ -87,34 +87,34 @@ avg diff: 0.001011
 min diff: 0.000000
 max diff: 0.004229
 gbrpf32le -> gbrp9le -> gbrpf32le
-avg diff: 0.003917
+avg diff: 0.000545
 min diff: 0.000000
-max diff: 0.007870
+max diff: 0.002245
 gbrpf32le -> gbrp10le -> gbrpf32le
-avg diff: 0.003841
+avg diff: 0.000350
 min diff: 0.000000
-max diff: 0.007456
+max diff: 0.001475
 gbrpf32le -> gbrap10le -> gbrpf32le
-avg diff: 0.003841
+avg diff: 0.000350
 min diff: 0.000000
-max diff: 0.007456
+max diff: 0.001475
 gbrpf32le -> gbrp12le -> gbrpf32le
-avg diff: 0.003796
+avg diff: 0.000260
 min diff: 0.000000
-max diff: 0.007140
+max diff: 0.001135
 gbrpf32le -> gbrap12le -> gbrpf32le
-avg diff: 0.003796
+avg diff: 0.000260
 min diff: 0.000000
-max diff: 0.007140
+max diff: 0.001135
 gbrpf32le -> gbrp14le -> gbrpf32le
-avg diff: 0.003792
+avg diff: 0.000253
 min diff: 0.000000
-max diff: 0.007034
+max diff: 0.001068
 gbrpf32le -> gbrp16le -> gbrpf32le
-avg diff: 0.003853
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007098
+max diff: 0.000990
 gbrpf32le -> gbrap16le -> gbrpf32le
-avg diff: 0.003853
+avg diff: 0.000249
 min diff: 0.000000
-max diff: 0.007098
+max diff: 0.000990
diff --git a/tests/ref/vsynth/vsynth1-ffv1-v3-rgb48 b/tests/ref/vsynth/vsynth1-ffv1-v3-rgb48
index 5c44488312..6f67de82b8 100644
--- a/tests/ref/vsynth/vsynth1-ffv1-v3-rgb48
+++ b/tests/ref/vsynth/vsynth1-ffv1-v3-rgb48
@@ -1,4 +1,4 @@
 f457dba7d58f0c28aedcfef518758d23 *tests/data/fate/vsynth1-ffv1-v3-rgb48.avi
 16858304 tests/data/fate/vsynth1-ffv1-v3-rgb48.avi
-b91ddc45eb70b4377dc7439e16733232 *tests/data/fate/vsynth1-ffv1-v3-rgb48.out.rawvideo
-stddev:    3.58 PSNR: 37.04 MAXDIFF:   75 bytes:  7603200/  7603200
+e2542b84193de462b04da530ab941f31 *tests/data/fate/vsynth1-ffv1-v3-rgb48.out.rawvideo
+stddev:    3.52 PSNR: 37.19 MAXDIFF:   74 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth1-r210 b/tests/ref/vsynth/vsynth1-r210
index 1103308060..2c62ef9779 100644
--- a/tests/ref/vsynth/vsynth1-r210
+++ b/tests/ref/vsynth/vsynth1-r210
@@ -1,4 +1,4 @@
 1a522a30ddd8c2865a731a5659001717 *tests/data/fate/vsynth1-r210.avi
 22125252 tests/data/fate/vsynth1-r210.avi
-b6444935d6c4d8c75fe63d5978f5b457 *tests/data/fate/vsynth1-r210.out.rawvideo
-stddev:    3.73 PSNR: 36.68 MAXDIFF:   48 bytes:  7603200/  7603200
+d118531accba75fb3b6ebf2a3a4a2e8e *tests/data/fate/vsynth1-r210.out.rawvideo
+stddev:    3.70 PSNR: 36.77 MAXDIFF:   47 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-ffv1-v3-rgb48 b/tests/ref/vsynth/vsynth2-ffv1-v3-rgb48
index 983c381e93..82b03bbe9c 100644
--- a/tests/ref/vsynth/vsynth2-ffv1-v3-rgb48
+++ b/tests/ref/vsynth/vsynth2-ffv1-v3-rgb48
@@ -1,4 +1,4 @@
 99b39b0286428ccf9c788545dfccb49f *tests/data/fate/vsynth2-ffv1-v3-rgb48.avi
 15089212 tests/data/fate/vsynth2-ffv1-v3-rgb48.avi
-b7278d99ea84fe8becf02de773fee0a5 *tests/data/fate/vsynth2-ffv1-v3-rgb48.out.rawvideo
-stddev:    1.43 PSNR: 44.97 MAXDIFF:   28 bytes:  7603200/  7603200
+370aa3829b8e828e97ec777f04375b26 *tests/data/fate/vsynth2-ffv1-v3-rgb48.out.rawvideo
+stddev:    1.24 PSNR: 46.23 MAXDIFF:   28 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-r210 b/tests/ref/vsynth/vsynth2-r210
index 5efd87d6f5..288664d693 100644
--- a/tests/ref/vsynth/vsynth2-r210
+++ b/tests/ref/vsynth/vsynth2-r210
@@ -1,4 +1,4 @@
 9a27c0c96f9e658d610d2590b61416a1 *tests/data/fate/vsynth2-r210.avi
 22125252 tests/data/fate/vsynth2-r210.avi
-d43196c64fd611f6e9c046e0ef3e570e *tests/data/fate/vsynth2-r210.out.rawvideo
-stddev:    1.37 PSNR: 45.34 MAXDIFF:   14 bytes:  7603200/  7603200
+8c370aac5d744c870049aed0741e20a3 *tests/data/fate/vsynth2-r210.out.rawvideo
+stddev:    1.20 PSNR: 46.54 MAXDIFF:   14 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth3-ffv1-v3-rgb48 b/tests/ref/vsynth/vsynth3-ffv1-v3-rgb48
index 9a7b6f5362..6964197e3e 100644
--- a/tests/ref/vsynth/vsynth3-ffv1-v3-rgb48
+++ b/tests/ref/vsynth/vsynth3-ffv1-v3-rgb48
@@ -1,4 +1,4 @@
 3bec2f4381dc362c24d1bbc304d0af58 *tests/data/fate/vsynth3-ffv1-v3-rgb48.avi
 244742 tests/data/fate/vsynth3-ffv1-v3-rgb48.avi
-d37e1507abd3233e27b0b00441149ddc *tests/data/fate/vsynth3-ffv1-v3-rgb48.out.rawvideo
-stddev:    3.52 PSNR: 37.19 MAXDIFF:   59 bytes:    86700/    86700
+a3dd456a61aa1abcf4fc2bc9ffd0617a *tests/data/fate/vsynth3-ffv1-v3-rgb48.out.rawvideo
+stddev:    3.48 PSNR: 37.30 MAXDIFF:   59 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth3-r210 b/tests/ref/vsynth/vsynth3-r210
index 253657cd85..ba5bb0b419 100644
--- a/tests/ref/vsynth/vsynth3-r210
+++ b/tests/ref/vsynth/vsynth3-r210
@@ -1,4 +1,4 @@
 fd12f6dde75d0872ccf9012b342208de *tests/data/fate/vsynth3-r210.avi
 442052 tests/data/fate/vsynth3-r210.avi
-a2c4e460ebede1109bd794b1b7b05a1f *tests/data/fate/vsynth3-r210.out.rawvideo
-stddev:    4.10 PSNR: 35.87 MAXDIFF:   48 bytes:    86700/    86700
+e09fcdcbbd4a247c7ad16b74eecb1ec7 *tests/data/fate/vsynth3-r210.out.rawvideo
+stddev:    4.07 PSNR: 35.93 MAXDIFF:   47 bytes:    86700/    86700
diff --git a/tests/ref/vsynth/vsynth_lena-ffv1-v3-rgb48 b/tests/ref/vsynth/vsynth_lena-ffv1-v3-rgb48
index 629d6850ad..e1b0885ece 100644
--- a/tests/ref/vsynth/vsynth_lena-ffv1-v3-rgb48
+++ b/tests/ref/vsynth/vsynth_lena-ffv1-v3-rgb48
@@ -1,4 +1,4 @@
 294d4f9fdf53ba0fe9d224d28ba700f0 *tests/data/fate/vsynth_lena-ffv1-v3-rgb48.avi
 13795036 tests/data/fate/vsynth_lena-ffv1-v3-rgb48.avi
-7bd82d1860095ae1e10ec0ace9cb04b6 *tests/data/fate/vsynth_lena-ffv1-v3-rgb48.out.rawvideo
-stddev:    0.91 PSNR: 48.91 MAXDIFF:   21 bytes:  7603200/  7603200
+832a42fafb18d52b35422ae0f34fe48c *tests/data/fate/vsynth_lena-ffv1-v3-rgb48.out.rawvideo
+stddev:    0.66 PSNR: 51.63 MAXDIFF:   21 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth_lena-r210 b/tests/ref/vsynth/vsynth_lena-r210
index 0a113dc693..f57d64db60 100644
--- a/tests/ref/vsynth/vsynth_lena-r210
+++ b/tests/ref/vsynth/vsynth_lena-r210
@@ -1,4 +1,4 @@
 61fd53566d99b725e75212747b35893f *tests/data/fate/vsynth_lena-r210.avi
 22125252 tests/data/fate/vsynth_lena-r210.avi
-4b7425191bb6a7fc4ca0dc649d9ba202 *tests/data/fate/vsynth_lena-r210.out.rawvideo
-stddev:    0.93 PSNR: 48.72 MAXDIFF:   11 bytes:  7603200/  7603200
+8ecccadf5147bd5e7158afef5e155413 *tests/data/fate/vsynth_lena-r210.out.rawvideo
+stddev:    0.75 PSNR: 50.57 MAXDIFF:   12 bytes:  7603200/  7603200