arm: looprestoration: NEON optimized wiener filter

The relative speedup compared to C code is around 4-8x:

                    Cortex A7     A8     A9    A53    A72    A73
wiener_luma_8bpc_neon:   4.00   7.54   4.74   6.84   4.91   8.01

1 files changed, 687 insertions, 0 deletions

diff --git a/src/arm/32/looprestoration.S b/src/arm/32/looprestoration.S
new file mode 100644
index 0000000..78fc96a
--- /dev/null
+++ b/src/arm/32/looprestoration.S
@@ -0,0 +1,687 @@
+/*
+ * Copyright © 2018, VideoLAN and dav1d authors
+ * Copyright © 2019, Martin Storsjo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "src/arm/asm.S"
+
+// void dav1d_wiener_filter_h_neon(int16_t *dst, const pixel (*left)[4],
+//                                 const pixel *src, ptrdiff_t stride,
+//                                 const int16_t fh[7], const intptr_t w,
+//                                 int h, enum LrEdgeFlags edges);
+function wiener_filter_h_neon, export=1
+        push            {r4-r11,lr}
+        vpush           {q4}
+        ldrd            r4,  r5,  [sp, #52]
+        ldrd            r6,  r7,  [sp, #60]
+        mov             r8,  r5
+        vld1.16         {q0},  [r4]
+        movw            r9,  #(1 << 14) - (1 << 2)
+        vdup.16         q14,  r9
+        vmov.s16        q15,  #2048
+        // Calculate mid_stride
+        add             r10, r5,  #7
+        bic             r10, r10, #7
+        lsl             r10, r10, #1
+
+        // Clear the last unused element of q0, to allow filtering a single
+        // pixel with one plain mul+addv.
+        mov             r12, #0
+        vmov.16         d1[3], r12
+
+        // Set up pointers for reading/writing alternate rows
+        add             r12, r0,  r10
+        lsl             r10, r10, #1
+        add             lr,  r2,  r3
+        lsl             r3,  r3,  #1
+
+        // Subtract the width from mid_stride
+        sub             r10, r10, r5, lsl #1
+
+        // For w >= 8, we read (w+5)&~7+8 pixels, for w < 8 we read 16 pixels.
+        cmp             r5,  #8
+        add             r11, r5,  #13
+        bic             r11, r11, #7
+        bge             1f
+        mov             r11, #16
+1:
+        sub             r3,  r3,  r11
+
+        // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL
+        tst             r7,  #1 // LR_HAVE_LEFT
+        beq             2f
+        // LR_HAVE_LEFT
+        cmp             r1,  #0
+        bne             0f
+        // left == NULL
+        sub             r2,  r2,  #3
+        sub             lr,  lr,  #3
+        b               1f
+0:      // LR_HAVE_LEFT, left != NULL
+2:      // !LR_HAVE_LEFT, increase the stride.
+        // For this case we don't read the left 3 pixels from the src pointer,
+        // but shift it as if we had done that.
+        add             r3,  r3,  #3
+
+
+1:      // Loop vertically
+        vld1.8          {q2},  [r2]!
+        vld1.8          {q9},  [lr]!
+
+        tst             r7,  #1 // LR_HAVE_LEFT
+        beq             0f
+        cmp             r1,  #0
+        beq             2f
+        // LR_HAVE_LEFT, left != NULL
+        vld1.32         {d3[1]},  [r1]!
+        // Move r2/lr back to account for the last 3 bytes we loaded earlier,
+        // which we'll shift out.
+        sub             r2,  r2,  #3
+        sub             lr,  lr,  #3
+        vld1.32         {d17[1]},  [r1]!
+        vext.8          q2,  q1,  q2,  #13
+        vext.8          q9,  q8,  q9,  #13
+        b               2f
+0:
+        // !LR_HAVE_LEFT, fill q1 with the leftmost byte
+        // and shift q2 to have 3x the first byte at the front.
+        vdup.8          q1, d4[0]
+        vdup.8          q8, d18[0]
+        // Move r2 back to account for the last 3 bytes we loaded before,
+        // which we shifted out.
+        sub             r2,  r2,  #3
+        sub             lr,  lr,  #3
+        vext.8          q2,  q1,  q2,  #13
+        vext.8          q9,  q8,  q9,  #13
+
+2:
+        vmovl.u8        q1,  d4
+        vmovl.u8        q2,  d5
+        vmovl.u8        q8,  d18
+        vmovl.u8        q9,  d19
+
+        tst             r7,  #2 // LR_HAVE_RIGHT
+        bne             4f
+        // If we'll need to pad the right edge, load that byte to pad with
+        // here since we can find it pretty easily from here.
+        sub             r9,  r5, #14
+        ldrb            r11, [r2, r9]
+        ldrb            r9,  [lr, r9]
+        // Fill q12/q13 with the right padding pixel
+        vdup.8          d24, r11
+        vdup.8          d26, r9
+        vmovl.u8        q12, d24
+        vmovl.u8        q13, d26
+3:      // !LR_HAVE_RIGHT
+        // If we'll have to pad the right edge we need to quit early here.
+        cmp             r5,  #11
+        bge             4f   // If w >= 11, all used input pixels are valid
+        cmp             r5,  #7
+        bge             5f   // If w >= 7, we can filter 4 pixels
+        b               6f
+
+4:      // Loop horizontally
+.macro filter_8
+        // This is tuned as some sort of compromise between Cortex A7, A8,
+        // A9 and A53.
+        vmul.s16        q3,  q1,  d0[0]
+        vext.8          q10, q1,  q2,  #2
+        vext.8          q11, q1,  q2,  #4
+        vmla.s16        q3,  q10, d0[1]
+        vmla.s16        q3,  q11, d0[2]
+        vext.8          q10, q1,  q2,  #6
+        vext.8          q11, q1,  q2,  #8
+        vmla.s16        q3,  q10, d0[3]
+        vmla.s16        q3,  q11, d1[0]
+        vext.8          q10, q1,  q2,  #10
+        vext.8          q11, q1,  q2,  #12
+        vmla.s16        q3,  q10, d1[1]
+        vmla.s16        q3,  q11, d1[2]
+
+        vmul.s16        q10, q8,  d0[0]
+        vext.8          q11, q8,  q9,  #2
+        vext.8          q4,  q8,  q9,  #4
+        vmla.s16        q10, q11, d0[1]
+        vmla.s16        q10, q4,  d0[2]
+        vext.8          q11, q8,  q9,  #6
+        vext.8          q4,  q8,  q9,  #8
+        vmla.s16        q10, q11, d0[3]
+        vmla.s16        q10, q4,  d1[0]
+        vext.8          q11, q8,  q9,  #10
+        vext.8          q4,  q8,  q9,  #12
+        vmla.s16        q10, q11, d1[1]
+        vmla.s16        q10, q4,  d1[2]
+
+        vext.8          q1,  q1,  q2,  #6
+        vext.8          q8,  q8,  q9,  #6
+        vshl.s16        q1,  q1,  #7
+        vshl.s16        q8,  q8,  #7
+        vsub.s16        q1,  q1,  q14
+        vsub.s16        q8,  q8,  q14
+        vqadd.s16       q3,  q3,  q1
+        vqadd.s16       q10, q10, q8
+        vshr.s16        q3,  q3,  #3
+        vshr.s16        q10, q10, #3
+        vadd.s16        q3,  q3,  q15
+        vadd.s16        q10, q10, q15
+.endm
+        filter_8
+        vst1.16         {q3},  [r0,  :128]!
+        vst1.16         {q10}, [r12, :128]!
+
+        subs            r5,  r5,  #8
+        ble             9f
+        tst             r7,  #2 // LR_HAVE_RIGHT
+        vmov            q1,  q2
+        vmov            q8,  q9
+        vld1.8          {d4},  [r2]!
+        vld1.8          {d18}, [lr]!
+        vmovl.u8        q2,  d4
+        vmovl.u8        q9,  d18
+        bne             4b // If we don't need to pad, just keep filtering.
+        b               3b // If we need to pad, check how many pixels we have left.
+
+5:      // Filter 4 pixels, 7 <= w < 11
+.macro filter_4
+        vmul.s16        d6,  d2,  d0[0]
+        vext.8          q10, q1,  q2,  #2
+        vext.8          q11, q1,  q2,  #4
+        vmla.s16        d6,  d20, d0[1]
+        vmla.s16        d6,  d22, d0[2]
+        vext.8          q10, q1,  q2,  #6
+        vext.8          q11, q1,  q2,  #8
+        vmla.s16        d6,  d20, d0[3]
+        vmla.s16        d6,  d22, d1[0]
+        vext.8          q10, q1,  q2,  #10
+        vext.8          q11, q1,  q2,  #12
+        vmla.s16        d6,  d20, d1[1]
+        vmla.s16        d6,  d22, d1[2]
+
+        vmul.s16        d20, d16, d0[0]
+        vext.8          q11, q8,  q9,  #2
+        vext.8          q4,  q8,  q9,  #4
+        vmla.s16        d20, d22, d0[1]
+        vmla.s16        d20, d8,  d0[2]
+        vext.8          q11, q8,  q9,  #6
+        vext.8          q4,  q8,  q9,  #8
+        vmla.s16        d20, d22, d0[3]
+        vmla.s16        d20, d8,  d1[0]
+        vext.8          q11, q8,  q9,  #10
+        vext.8          q4,  q8,  q9,  #12
+        vmla.s16        d20, d22, d1[1]
+        vmla.s16        d20, d8,  d1[2]
+
+        vext.8          q11, q1,  q2,  #6
+        vshl.s16        d22, d22, #7
+        vsub.s16        d22, d22, d28
+        vqadd.s16       d6,  d6,  d22
+        vext.8          q11, q8,  q9,  #6
+        vshl.s16        d22, d22, #7
+        vsub.s16        d22, d22, d28
+        vqadd.s16       d20, d20, d22
+        vshr.s16        d6,  d6,  #3
+        vshr.s16        d20, d20, #3
+        vadd.s16        d6,  d6,  d30
+        vadd.s16        d20, d20, d30
+.endm
+        filter_4
+        vst1.16         {d6},  [r0,  :64]!
+        vst1.16         {d20}, [r12, :64]!
+
+        subs            r5,  r5,  #4 // 3 <= w < 7
+        vext.8          q1,  q1,  q2,  #8
+        vext.8          q2,  q2,  q2,  #8
+        vext.8          q8,  q8,  q9,  #8
+        vext.8          q9,  q9,  q9,  #8
+
+6:      // Pad the right edge and filter the last few pixels.
+        // w < 7, w+3 pixels valid in q1-q2
+        cmp             r5,  #5
+        blt             7f
+        bgt             8f
+        // w == 5, 8 pixels valid in q1, q2 invalid
+        vmov            q2,  q12
+        vmov            q9,  q13
+        b               88f
+
+7:      // 1 <= w < 5, 4-7 pixels valid in q1
+        sub             r9,  r5,  #1
+        // w9 = (pixels valid - 4)
+        adr             r11, L(variable_shift_tbl)
+        ldr             r9,  [r11, r9, lsl #2]
+        add             r11, r11, r9
+        vmov            q2,  q12
+        vmov            q9,  q13
+        bx              r11
+
+        .align 2
+L(variable_shift_tbl):
+        .word 44f - L(variable_shift_tbl) + CONFIG_THUMB
+        .word 55f - L(variable_shift_tbl) + CONFIG_THUMB
+        .word 66f - L(variable_shift_tbl) + CONFIG_THUMB
+        .word 77f - L(variable_shift_tbl) + CONFIG_THUMB
+
+        // Shift q1 right, shifting out invalid pixels,
+        // shift q1 left to the original offset, shifting in padding pixels.
+44:     // 4 pixels valid
+        vext.8          q1,  q1,  q1,  #8
+        vext.8          q1,  q1,  q2,  #8
+        vext.8          q8,  q8,  q8,  #8
+        vext.8          q8,  q8,  q9,  #8
+        b               88f
+55:     // 5 pixels valid
+        vext.8          q1,  q1,  q1,  #10
+        vext.8          q1,  q1,  q2,  #6
+        vext.8          q8,  q8,  q8,  #10
+        vext.8          q8,  q8,  q9,  #6
+        b               88f
+66:     // 6 pixels valid
+        vext.8          q1,  q1,  q1,  #12
+        vext.8          q1,  q1,  q2,  #4
+        vext.8          q8,  q8,  q8,  #12
+        vext.8          q8,  q8,  q9,  #4
+        b               88f
+77:     // 7 pixels valid
+        vext.8          q1,  q1,  q1,  #14
+        vext.8          q1,  q1,  q2,  #2
+        vext.8          q8,  q8,  q8,  #14
+        vext.8          q8,  q8,  q9,  #2
+        b               88f
+
+8:      // w > 5, w == 6, 9 pixels valid in q1-q2, 1 pixel valid in q2
+        vext.8          q2,  q2,  q2,  #2
+        vext.8          q2,  q2,  q12, #14
+        vext.8          q9,  q9,  q9,  #2
+        vext.8          q9,  q9,  q13, #14
+
+88:
+        // w < 7, q1-q2 padded properly
+        cmp             r5,  #4
+        blt             888f
+
+        // w >= 4, filter 4 pixels
+        filter_4
+        vst1.16         {d6},  [r0,  :64]!
+        vst1.16         {d20}, [r12, :64]!
+        subs            r5,  r5,  #4 // 0 <= w < 4
+        vext.8          q1,  q1,  q2,  #8
+        vext.8          q8,  q8,  q9,  #8
+        beq             9f
+888:    // 1 <= w < 4, filter 1 pixel at a time
+        vmul.s16        q3,  q1,  q0
+        vmul.s16        q10, q8,  q0
+        vpadd.s16       d6,  d6,  d7
+        vpadd.s16       d7,  d20, d21
+        vdup.16         d24, d2[3]
+        vpadd.s16       d6,  d6,  d7
+        vdup.16         d25, d16[3]
+        vpadd.s16       d6,  d6,  d6
+        vtrn.16         d24, d25
+        vshl.s16        d24, d24,  #7
+        vsub.s16        d24, d24,  d28
+        vqadd.s16       d6,  d6,   d24
+        vshr.s16        d6,  d6,   #3
+        vadd.s16        d6,  d6,   d30
+        vst1.s16        {d6[0]}, [r0,  :16]!
+        vst1.s16        {d6[1]}, [r12, :16]!
+        subs            r5,  r5,  #1
+        vext.8          q1,  q1,  q2,  #2
+        vext.8          q8,  q8,  q9,  #2
+        bgt             888b
+
+9:
+        subs            r6,  r6,  #2
+        ble             0f
+        // Jump to the next row and loop horizontally
+        add             r0,  r0,  r10
+        add             r12, r12, r10
+        add             r2,  r2,  r3
+        add             lr,  lr,  r3
+        mov             r5,  r8
+        b               1b
+0:
+        vpop            {q4}
+        pop             {r4-r11,pc}
+.purgem filter_8
+.purgem filter_4
+endfunc
+
+// void dav1d_wiener_filter_v_neon(pixel *dst, ptrdiff_t stride,
+//                                 const int16_t *mid, int w, int h,
+//                                 const int16_t fv[7], enum LrEdgeFlags edges,
+//                                 ptrdiff_t mid_stride);
+function wiener_filter_v_neon, export=1
+        push            {r4-r7,lr}
+        ldrd            r4,  r5,  [sp, #20]
+        ldrd            r6,  r7,  [sp, #28]
+        mov             lr,  r4
+        vmov.s16        q1,  #0
+        mov             r12, #128
+        vld1.16         {q0},  [r5]
+        vmov.s16        d2[3], r12
+        vadd.s16        q0,  q0,  q1
+
+        // Calculate the number of rows to move back when looping vertically
+        mov             r12, r4
+        tst             r6,  #4 // LR_HAVE_TOP
+        beq             0f
+        sub             r2,  r2,  r7,  lsl #1
+        add             r12, r12, #2
+0:
+        tst             r6,  #8 // LR_HAVE_BOTTOM
+        beq             1f
+        add             r12, r12, #2
+
+1:      // Start of horizontal loop; start one vertical filter slice.
+        // Load rows into q8-q11 and pad properly.
+        tst             r6,  #4 // LR_HAVE_TOP
+        vld1.16         {q8},  [r2, :128], r7
+        beq             2f
+        // LR_HAVE_TOP
+        vld1.16         {q10}, [r2, :128], r7
+        vmov            q9,  q8
+        vld1.16         {q11}, [r2, :128], r7
+        b               3f
+2:      // !LR_HAVE_TOP
+        vmov            q9,  q8
+        vmov            q10, q8
+        vmov            q11, q8
+
+3:
+        cmp             r4,  #4
+        blt             5f
+        // Start filtering normally; fill in q12-q14 with unique rows.
+        vld1.16         {q12}, [r2, :128], r7
+        vld1.16         {q13}, [r2, :128], r7
+        vld1.16         {q14}, [r2, :128], r7
+
+4:
+.macro filter compare
+        subs            r4,  r4,  #1
+        // Interleaving the mul/mla chains actually hurts performance
+        // significantly on Cortex A53, thus keeping mul/mla tightly
+        // chained like this.
+        vmull.s16       q2,  d16,  d0[0]
+        vmlal.s16       q2,  d18,  d0[1]
+        vmlal.s16       q2,  d20,  d0[2]
+        vmlal.s16       q2,  d22,  d0[3]
+        vmlal.s16       q2,  d24,  d1[0]
+        vmlal.s16       q2,  d26,  d1[1]
+        vmlal.s16       q2,  d28,  d1[2]
+        vmull.s16       q3,  d17,  d0[0]
+        vmlal.s16       q3,  d19,  d0[1]
+        vmlal.s16       q3,  d21,  d0[2]
+        vmlal.s16       q3,  d23,  d0[3]
+        vmlal.s16       q3,  d25,  d1[0]
+        vmlal.s16       q3,  d27,  d1[1]
+        vmlal.s16       q3,  d29,  d1[2]
+        vqrshrun.s32    d4,  q2,   #11
+        vqrshrun.s32    d5,  q3,   #11
+        vqmovun.s16     d4,  q2
+        vst1.8          {d4}, [r0], r1
+.if \compare
+        cmp             r4,  #4
+.else
+        ble             9f
+.endif
+        vmov            q8,  q9
+        vmov            q9,  q10
+        vmov            q10, q11
+        vmov            q11, q12
+        vmov            q12, q13
+        vmov            q13, q14
+.endm
+        filter          1
+        blt             7f
+        vld1.16         {q14}, [r2, :128], r7
+        b               4b
+
+5:      // Less than 4 rows in total; not all of q12-q13 are filled yet.
+        tst             r6,  #8 // LR_HAVE_BOTTOM
+        beq             6f
+        // LR_HAVE_BOTTOM
+        cmp             r4,  #2
+        // We load at least 2 rows in all cases.
+        vld1.16         {q12}, [r2, :128], r7
+        vld1.16         {q13}, [r2, :128], r7
+        bgt             53f // 3 rows in total
+        beq             52f // 2 rows in total
+51:     // 1 row in total, q11 already loaded, load edge into q12-q14.
+        vmov            q13, q12
+        b               8f
+52:     // 2 rows in total, q11 already loaded, load q12 with content data
+        // and 2 rows of edge.
+        vld1.16         {q14}, [r2, :128], r7
+        vmov            q15,  q14
+        b               8f
+53:
+        // 3 rows in total, q11 already loaded, load q12 and q13 with content
+        // and 2 rows of edge.
+        vld1.16         {q14}, [r2, :128], r7
+        vld1.16         {q15}, [r2, :128], r7
+        vmov            q1,  q15
+        b               8f
+
+6:
+        // !LR_HAVE_BOTTOM
+        cmp             r4,  #2
+        bgt             63f // 3 rows in total
+        beq             62f // 2 rows in total
+61:     // 1 row in total, q11 already loaded, pad that into q12-q14.
+        vmov            q12, q11
+        vmov            q13, q11
+        vmov            q14, q11
+        b               8f
+62:     // 2 rows in total, q11 already loaded, load q12 and pad that into q12-q15.
+        vld1.16         {q12}, [r2, :128], r7
+        vmov            q13, q12
+        vmov            q14, q12
+        vmov            q15, q12
+        b               8f
+63:
+        // 3 rows in total, q11 already loaded, load q12 and q13 and pad q13 into q14-q15,q1.
+        vld1.16         {q12}, [r2, :128], r7
+        vld1.16         {q13}, [r2, :128], r7
+        vmov            q14, q13
+        vmov            q15, q13
+        vmov            q1,  q13
+        b               8f
+
+7:
+        // All registers up to q13 are filled already, 3 valid rows left.
+        // < 4 valid rows left; fill in padding and filter the last
+        // few rows.
+        tst             r6,  #8 // LR_HAVE_BOTTOM
+        beq             71f
+        // LR_HAVE_BOTTOM; load 2 rows of edge.
+        vld1.16         {q14}, [r2, :128], r7
+        vld1.16         {q15}, [r2, :128], r7
+        vmov            q1,  q15
+        b               8f
+71:
+        // !LR_HAVE_BOTTOM, pad 3 rows
+        vmov            q14, q13
+        vmov            q15, q13
+        vmov            q1,  q13
+
+8:      // At this point, all registers up to q14-15,q1 are loaded with
+        // edge/padding (depending on how many rows are left).
+        filter          0 // This branches to 9f when done
+        vmov            q13, q14
+        vmov            q14, q1
+        b               8b
+
+9:      // End of one vertical slice.
+        subs            r3,  r3,  #8
+        ble             0f
+        // Move pointers back up to the top and loop horizontally.
+        mls             r0,  r1,  lr,  r0
+        mls             r2,  r7,  r12, r2
+        add             r0,  r0,  #8
+        add             r2,  r2,  #16
+        mov             r4,  lr
+        b               1b
+
+0:
+        pop             {r4-r7,pc}
+.purgem filter
+endfunc
+
+// void dav1d_copy_narrow_neon(pixel *dst, ptrdiff_t stride,
+//                             const pixel *src, int w, int h);
+function copy_narrow_neon, export=1
+        push            {r4,lr}
+        ldr             r4, [sp, #8]
+        adr             r12, L(copy_narrow_tbl)
+        ldr             r3,  [r12, r3, lsl #2]
+        add             r12, r12, r3
+        bx              r12
+
+        .align 2
+L(copy_narrow_tbl):
+        .word 0
+        .word 10f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 20f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 30f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 40f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 50f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 60f - L(copy_narrow_tbl) + CONFIG_THUMB
+        .word 70f - L(copy_narrow_tbl) + CONFIG_THUMB
+
+10:
+        add             r3,  r0,  r1
+        lsl             r1,  r1,  #1
+18:
+        subs            r4,  r4,  #8
+        blt             110f
+        vld1.8          {d0}, [r2, :64]!
+        vst1.8          {d0[0]}, [r0], r1
+        vst1.8          {d0[1]}, [r3], r1
+        vst1.8          {d0[2]}, [r0], r1
+        vst1.8          {d0[3]}, [r3], r1
+        vst1.8          {d0[4]}, [r0], r1
+        vst1.8          {d0[5]}, [r3], r1
+        vst1.8          {d0[6]}, [r0], r1
+        vst1.8          {d0[7]}, [r3], r1
+        ble             0f
+        b               18b
+110:
+        add             r4,  r4,  #8
+        asr             r1,  r1,  #1
+11:
+        subs            r4,  r4,  #1
+        vld1.8          {d0[]},  [r2]!
+        vst1.8          {d0[0]}, [r0], r1
+        bgt             11b
+0:
+        pop             {r4,pc}
+
+20:
+        add             r3,  r0,  r1
+        lsl             r1,  r1,  #1
+24:
+        subs            r4,  r4,  #4
+        blt             210f
+        vld1.16         {d0}, [r2, :64]!
+        vst1.16         {d0[0]}, [r0, :16], r1
+        vst1.16         {d0[1]}, [r3, :16], r1
+        vst1.16         {d0[2]}, [r0, :16], r1
+        vst1.16         {d0[3]}, [r3, :16], r1
+        ble             0f
+        b               24b
+210:
+        add             r4,  r4,  #4
+        asr             r1,  r1,  #1
+22:
+        subs            r4,  r4,  #1
+        vld1.16         {d0[]},  [r2]!
+        vst1.16         {d0[0]}, [r0], r1
+        bgt             22b
+0:
+        pop             {r4,pc}
+
+30:
+        ldrh            r3,  [r2]
+        ldrb            r12, [r2, #2]
+        add             r2,  r2,  #3
+        subs            r4,  r4,  #1
+        strh            r3,  [r0]
+        strb            r12, [r0, #2]
+        add             r0,  r0,  r1
+        bgt             30b
+        pop             {r4,pc}
+
+40:
+        add             r3,  r0,  r1
+        lsl             r1,  r1,  #1
+42:
+        subs            r4,  r4,  #2
+        blt             41f
+        vld1.8          {d0}, [r2, :64]!
+        vst1.32         {d0[0]}, [r0, :32], r1
+        vst1.32         {d0[1]}, [r3, :32], r1
+        ble             0f
+        b               42b
+41:
+        vld1.32         {d0[]},  [r2]
+        vst1.32         {d0[0]}, [r0]
+0:
+        pop             {r4,pc}
+
+50:
+        ldr             r3,  [r2]
+        ldrb            r12, [r2, #4]
+        add             r2,  r2,  #5
+        subs            r4,  r4,  #1
+        str             r3,  [r0]
+        strb            r12, [r0, #4]
+        add             r0,  r0,  r1
+        bgt             50b
+        pop             {r4,pc}
+
+60:
+        ldr             r3,  [r2]
+        ldrh            r12, [r2, #4]
+        add             r2,  r2,  #6
+        subs            r4,  r4,  #1
+        str             r3,  [r0]
+        strh            r12, [r0, #4]
+        add             r0,  r0,  r1
+        bgt             60b
+        pop             {r4,pc}
+
+70:
+        ldr             r3,  [r2]
+        ldrh            r12, [r2, #4]
+        ldrb            lr,  [r2, #6]
+        add             r2,  r2,  #7
+        subs            r4,  r4,  #1
+        str             r3,  [r0]
+        strh            r12, [r0, #4]
+        strb            lr,  [r0, #6]
+        add             r0,  r0,  r1
+        bgt             70b
+        pop             {r4,pc}
+endfunc