Simplify/rewrite cdef filtering code

1 files changed, 67 insertions, 86 deletions

diff --git a/src/cdef_tmpl.c b/src/cdef_tmpl.c
index 0f6aada..7cd1be6 100644
--- a/src/cdef_tmpl.c
+++ b/src/cdef_tmpl.c
@@ -25,17 +25,6 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-/*
- * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved
- *
- * This source code is subject to the terms of the BSD 2 Clause License and
- * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
- * was not distributed with this source code in the LICENSE file, you can
- * obtain it at www.aomedia.org/license/software. If the Alliance for Open
- * Media Patent License 1.0 was not distributed with this source code in the
- * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
- */
-
 #include "config.h"
 
 #include <assert.h>
@@ -45,26 +34,15 @@
 
 #include "src/cdef.h"
 
-static const int8_t cdef_directions4[8 /* dir */][2 /* pass */] = {
-    { -1 * 8 + 1, -2 * 8 + 2 },
-    {  0 * 8 + 1, -1 * 8 + 2 },
-    {  0 * 8 + 1,  0 * 8 + 2 },
-    {  0 * 8 + 1,  1 * 8 + 2 },
-    {  1 * 8 + 1,  2 * 8 + 2 },
-    {  1 * 8 + 0,  2 * 8 + 1 },
-    {  1 * 8 + 0,  2 * 8 + 0 },
-    {  1 * 8 + 0,  2 * 8 - 1 }
-};
-
-static const int8_t cdef_directions8[8 /* dir */][2 /* pass */] = {
-    { -1 * 16 + 1, -2 * 16 + 2 },
-    {  0 * 16 + 1, -1 * 16 + 2 },
-    {  0 * 16 + 1,  0 * 16 + 2 },
-    {  0 * 16 + 1,  1 * 16 + 2 },
-    {  1 * 16 + 1,  2 * 16 + 2 },
-    {  1 * 16 + 0,  2 * 16 + 1 },
-    {  1 * 16 + 0,  2 * 16 + 0 },
-    {  1 * 16 + 0,  2 * 16 - 1 }
+static const int8_t cdef_directions[8 /* dir */][2 /* pass */] = {
+    { -1 * 12 + 1, -2 * 12 + 2 },
+    {  0 * 12 + 1, -1 * 12 + 2 },
+    {  0 * 12 + 1,  0 * 12 + 2 },
+    {  0 * 12 + 1,  1 * 12 + 2 },
+    {  1 * 12 + 1,  2 * 12 + 2 },
+    {  1 * 12 + 0,  2 * 12 + 1 },
+    {  1 * 12 + 0,  2 * 12 + 0 },
+    {  1 * 12 + 0,  2 * 12 - 1 }
 };
 static const uint8_t cdef_pri_taps[2][2] = { { 4, 2 }, { 3, 3 } };
 static const uint8_t cdef_sec_taps[2][2] = { { 2, 1 }, { 2, 1 } };
@@ -78,92 +56,97 @@ static inline int constrain(const int diff, const int threshold,
                       diff);
 }
 
-/*
- * <code partially copied from libaom>
- */
-
-#define CDEF_VERY_LARGE (30000)
-
-static void fill(uint16_t *tmp, const ptrdiff_t stride,
-                 const int w, const int h)
+static inline void fill(uint16_t *tmp, const ptrdiff_t stride,
+                        const int w, const int h)
 {
     for (int y = 0; y < h; y++) {
         for (int x = 0; x < w; x++)
-            tmp[x] = CDEF_VERY_LARGE;
+            tmp[x] = INT16_MAX;
         tmp += stride;
     }
 }
 
-/* Smooth in the direction detected. */
-static void cdef_filter_block_c(pixel *const dst, const ptrdiff_t dst_stride,
-                                /*const*/ pixel *const top[2],
-                                const int w, const int h, const int pri_strength,
-                                const int sec_strength, const int dir,
-                                const int damping, const enum CdefEdgeFlags edges)
+static void padding(uint16_t *tmp, const ptrdiff_t tmp_stride,
+                    const pixel *src, const ptrdiff_t src_stride,
+                    const pixel (*left)[2], pixel *const top[2],
+                    const int w, const int h,
+                    const enum CdefEdgeFlags edges)
 {
-    const ptrdiff_t tmp_stride = 16 >> (w == 4);
-    assert((w == 4 || w == 8) && (h == 4 || h == 8));
-    uint16_t tmp[192];  // 16*12 is the maximum value of tmp_stride * (h + 4)
-    uint16_t *tmp2 = tmp + 2 * tmp_stride + 2;
-    const uint8_t *const pri_taps = cdef_pri_taps[(pri_strength >> (BITDEPTH - 8)) & 1];
-    const uint8_t *const sec_taps = cdef_sec_taps[(pri_strength >> (BITDEPTH - 8)) & 1];
-    const int8_t (*cdef_directions)[2];
-
-    assert(w == 4 || w == 8);
-    cdef_directions = w == 4 ? cdef_directions4 : cdef_directions8;
-
     // fill extended input buffer
     int x_start = -2, x_end = w + 2, y_start = -2, y_end = h + 2;
     if (!(edges & HAVE_TOP)) {
-        fill(tmp, tmp_stride, w + 4, 2);
+        fill(tmp - 2 - 2 * tmp_stride, tmp_stride, w + 4, 2);
         y_start = 0;
     }
     if (!(edges & HAVE_BOTTOM)) {
-        fill(tmp + (h + 2) * tmp_stride, tmp_stride, w + 4, 2);
+        fill(tmp + h * tmp_stride - 2, tmp_stride, w + 4, 2);
         y_end -= 2;
     }
     if (!(edges & HAVE_LEFT)) {
-        fill(tmp + (2 + y_start) * tmp_stride, tmp_stride, 2, y_end - y_start);
+        fill(tmp + y_start * tmp_stride - 2, tmp_stride, 2, y_end - y_start);
         x_start = 0;
     }
     if (!(edges & HAVE_RIGHT)) {
-        fill(tmp + (2 + y_start) * tmp_stride + w + 2, tmp_stride,
-             2, y_end - y_start);
+        fill(tmp + y_start * tmp_stride + w, tmp_stride, 2, y_end - y_start);
         x_end -= 2;
     }
+
     for (int y = y_start; y < 0; y++)
         for (int x = x_start; x < x_end; x++)
-            tmp2[y * tmp_stride + x] = top[y & 1][x];
-    for (int y = 0; y < y_end; y++)
-        for (int x = x_start; x < x_end; x++)
-            tmp2[y * tmp_stride + x] = dst[y * PXSTRIDE(dst_stride) + x];
+            tmp[x + y * tmp_stride] = top[y & 1][x];
+    for (int y = 0; y < h; y++)
+        for (int x = x_start; x < 0; x++)
+            tmp[x + y * tmp_stride] = left[y][2 + x];
+    for (int y = 0; y < y_end; y++) {
+        for (int x = (y < h) ? 0 : x_start; x < x_end; x++)
+            tmp[x] = src[x];
+        src += PXSTRIDE(src_stride);
+        tmp += tmp_stride;
+    }
+}
+
+static NOINLINE void
+cdef_filter_block_c(pixel *dst, const ptrdiff_t dst_stride,
+                    const pixel (*left)[2], /*const*/ pixel *const top[2],
+                    const int w, const int h, const int pri_strength,
+                    const int sec_strength, const int dir,
+                    const int damping, const enum CdefEdgeFlags edges)
+{
+    const ptrdiff_t tmp_stride = 12;
+    assert((w == 4 || w == 8) && (h == 4 || h == 8));
+    uint16_t tmp_buf[144];  // 12*12 is the maximum value of tmp_stride * (h + 4)
+    uint16_t *tmp = tmp_buf + 2 * tmp_stride + 2;
+    const uint8_t *const pri_taps = cdef_pri_taps[(pri_strength >> (BITDEPTH - 8)) & 1];
+    const uint8_t *const sec_taps = cdef_sec_taps[(pri_strength >> (BITDEPTH - 8)) & 1];
+
+    padding(tmp, tmp_stride, dst, dst_stride, left, top, w, h, edges);
 
     // run actual filter
     for (int y = 0; y < h; y++) {
         for (int x = 0; x < w; x++) {
             int sum = 0;
-            const int px = dst[y * PXSTRIDE(dst_stride) + x];
+            const int px = dst[x];
             int max = px, min = px;
             for (int k = 0; k < 2; k++) {
                 const int8_t off1 = cdef_directions[dir][k];
-                const int p0 = tmp2[y * tmp_stride + x + off1];
-                const int p1 = tmp2[y * tmp_stride + x - off1];
+                const int p0 = tmp[x + off1];
+                const int p1 = tmp[x - off1];
                 sum += pri_taps[k] * constrain(p0 - px, pri_strength, damping);
                 sum += pri_taps[k] * constrain(p1 - px, pri_strength, damping);
-                if (p0 != CDEF_VERY_LARGE) max = imax(p0, max);
-                if (p1 != CDEF_VERY_LARGE) max = imax(p1, max);
+                if (p0 != INT16_MAX) max = imax(p0, max);
+                if (p1 != INT16_MAX) max = imax(p1, max);
                 min = imin(p0, min);
                 min = imin(p1, min);
                 const int8_t off2 = cdef_directions[(dir + 2) & 7][k];
-                const int s0 = tmp2[y * tmp_stride + x + off2];
-                const int s1 = tmp2[y * tmp_stride + x - off2];
+                const int s0 = tmp[x + off2];
+                const int s1 = tmp[x - off2];
                 const int8_t off3 = cdef_directions[(dir + 6) & 7][k];
-                const int s2 = tmp2[y * tmp_stride + x + off3];
-                const int s3 = tmp2[y * tmp_stride + x - off3];
-                if (s0 != CDEF_VERY_LARGE) max = imax(s0, max);
-                if (s1 != CDEF_VERY_LARGE) max = imax(s1, max);
-                if (s2 != CDEF_VERY_LARGE) max = imax(s2, max);
-                if (s3 != CDEF_VERY_LARGE) max = imax(s3, max);
+                const int s2 = tmp[x + off3];
+                const int s3 = tmp[x - off3];
+                if (s0 != INT16_MAX) max = imax(s0, max);
+                if (s1 != INT16_MAX) max = imax(s1, max);
+                if (s2 != INT16_MAX) max = imax(s2, max);
+                if (s3 != INT16_MAX) max = imax(s3, max);
                 min = imin(s0, min);
                 min = imin(s1, min);
                 min = imin(s2, min);
@@ -173,19 +156,17 @@ static void cdef_filter_block_c(pixel *const dst, const ptrdiff_t dst_stride,
                 sum += sec_taps[k] * constrain(s2 - px, sec_strength, damping);
                 sum += sec_taps[k] * constrain(s3 - px, sec_strength, damping);
             }
-            dst[y * PXSTRIDE(dst_stride) + x] =
-                iclip(px + ((8 + sum - (sum < 0)) >> 4), min, max);
+            dst[x] = iclip(px + ((8 + sum - (sum < 0)) >> 4), min, max);
         }
+        dst += PXSTRIDE(dst_stride);
+        tmp += tmp_stride;
     }
 }
 
-/*
- * </code partially copied from libaom>
- */
-
 #define cdef_fn(w, h) \
 static void cdef_filter_block_##w##x##h##_c(pixel *const dst, \
                                             const ptrdiff_t stride, \
+                                            const pixel (*left)[2], \
                                             /*const*/ pixel *const top[2], \
                                             const int pri_strength, \
                                             const int sec_strength, \
@@ -193,7 +174,7 @@ static void cdef_filter_block_##w##x##h##_c(pixel *const dst, \
                                             const int damping, \
                                             const enum CdefEdgeFlags edges) \
 { \
-    cdef_filter_block_c(dst, stride, top, w, h, pri_strength, sec_strength, \
+    cdef_filter_block_c(dst, stride, left, top, w, h, pri_strength, sec_strength, \
                         dir, damping, edges); \
 }