path: root/src/x86/filmgrain16_avx2.asm

; Copyright © 2021-2022, VideoLAN and dav1d authors
; Copyright © 2021-2022, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"
%include "x86/filmgrain_common.asm"

%if ARCH_X86_64

SECTION_RODATA 16
pb_mask:       db  0,128,128,  0,128,  0,  0,128,128,  0,  0,128,  0,128,128,  0
gen_shufA:     db  0,  1,  2,  3,  2,  3,  4,  5,  4,  5,  6,  7,  6,  7,  8,  9
gen_shufB:     db  4,  5,  6,  7,  6,  7,  8,  9,  8,  9, 10, 11, 10, 11, 12, 13
next_upperbit_mask:    dw 0x100B, 0x2016, 0x402C, 0x8058
pw_27_17_17_27:        dw 27, 17, 17, 27
pw_23_22:              dw 23, 22, 0, 32
pw_seed_xor:   times 2 dw 0xb524
               times 2 dw 0x49d8
gen_ar0_shift: times 4 db 128
               times 4 db 64
               times 4 db 32
               times 4 db 16
pd_16:                 dd 16
pd_m65536:             dd -65536
pb_1:          times 4 db 1
grain_max:     times 2 dw  511
               times 2 dw 2047
grain_min:     times 2 dw -512
               times 2 dw -2048
fg_max:        times 2 dw 1023
               times 2 dw 4095
               times 2 dw 960
               times 2 dw 3840
               times 2 dw 940
               times 2 dw 3760
fg_min:        times 2 dw 0
               times 2 dw 64
               times 2 dw 256
uv_offset_mul:         dd 256
                       dd 1024
hmul_bits:             dw 32768, 16384,  8192,  4096
round:                 dw  2048,  1024,   512
mul_bits:              dw   256,   128,    64,    32,    16,     8
round_vals:            dw    32,    64,   128,   256,   512,  1024
pb_8_9_0_1:            db 8, 9, 0, 1

%macro JMP_TABLE 1-*
    %xdefine %1_table %%table
    %xdefine %%base %1_table
    %xdefine %%prefix mangle(private_prefix %+ _%1)
    %%table:
    %rep %0 - 1
        dd %%prefix %+ .ar%2 - %%base
        %rotate 1
    %endrep
%endmacro

JMP_TABLE generate_grain_y_16bpc_avx2, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_420_16bpc_avx2, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_422_16bpc_avx2, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_444_16bpc_avx2, 0, 1, 2, 3

SECTION .text

%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)

INIT_YMM avx2
cglobal generate_grain_y_16bpc, 3, 9, 14, buf, fg_data, bdmax
%define base r4-generate_grain_y_16bpc_avx2_table
    lea              r4, [generate_grain_y_16bpc_avx2_table]
    vpbroadcastw    xm0, [fg_dataq+FGData.seed]
    mov             r6d, [fg_dataq+FGData.grain_scale_shift]
    movq            xm1, [base+next_upperbit_mask]
    mov              r3, -73*82*2
    movsxd           r5, [fg_dataq+FGData.ar_coeff_lag]
    lea             r7d, [bdmaxq+1]
    movq            xm4, [base+mul_bits]
    shr             r7d, 11             ; 0 for 10bpc, 2 for 12bpc
    movq            xm5, [base+hmul_bits]
    sub              r6, r7
    mova            xm6, [base+pb_mask]
    sub            bufq, r3
    vpbroadcastw    xm7, [base+round+r6*2-2]
    lea              r6, [gaussian_sequence]
    movsxd           r5, [r4+r5*4]
.loop:
    pand            xm2, xm0, xm1
    psrlw           xm3, xm2, 10
    por             xm2, xm3            ; bits 0xf, 0x1e, 0x3c and 0x78 are set
    pmullw          xm2, xm4            ; bits 0x0f00 are set
    pmulhuw         xm0, xm5
    pshufb          xm3, xm6, xm2       ; set 15th bit for next 4 seeds
    psllq           xm2, xm3, 30
    por             xm2, xm3
    psllq           xm3, xm2, 15
    por             xm2, xm0            ; aggregate each bit into next seed's high bit
    por             xm3, xm2            ; 4 next output seeds
    pshuflw         xm0, xm3, q3333
    psrlw           xm3, 5
    pand            xm2, xm0, xm1
    movq             r7, xm3
    psrlw           xm3, xm2, 10
    por             xm2, xm3
    pmullw          xm2, xm4
    pmulhuw         xm0, xm5
    movzx           r8d, r7w
    pshufb          xm3, xm6, xm2
    psllq           xm2, xm3, 30
    por             xm2, xm3
    psllq           xm3, xm2, 15
    por             xm0, xm2
    movd            xm2, [r6+r8*2]
    rorx             r8, r7, 32
    por             xm3, xm0
    shr             r7d, 16
    pinsrw          xm2, [r6+r7*2], 1
    pshuflw         xm0, xm3, q3333
    movzx           r7d, r8w
    psrlw           xm3, 5
    pinsrw          xm2, [r6+r7*2], 2
    shr             r8d, 16
    movq             r7, xm3
    pinsrw          xm2, [r6+r8*2], 3
    movzx           r8d, r7w
    pinsrw          xm2, [r6+r8*2], 4
    rorx             r8, r7, 32
    shr             r7d, 16
    pinsrw          xm2, [r6+r7*2], 5
    movzx           r7d, r8w
    pinsrw          xm2, [r6+r7*2], 6
    shr             r8d, 16
    pinsrw          xm2, [r6+r8*2], 7
    paddw           xm2, xm2            ; otherwise bpc=12 w/ grain_scale_shift=0
    pmulhrsw        xm2, xm7            ; shifts by 0, which pmulhrsw does not support
    mova      [bufq+r3], xm2
    add              r3, 8*2
    jl .loop

    ; auto-regression code
    add              r5, r4
    jmp              r5

.ar1:
    DEFINE_ARGS buf, fg_data, max, shift, val3, min, cf3, x, val0
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movsx          cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3]
    movd            xm4, [fg_dataq+FGData.ar_coeffs_y]
    DEFINE_ARGS buf, h, max, shift, val3, min, cf3, x, val0
    pinsrb          xm4, [base+pb_1], 3
    pmovsxbw        xm4, xm4
    pshufd          xm5, xm4, q1111
    pshufd          xm4, xm4, q0000
    vpbroadcastw    xm3, [base+round_vals+shiftq*2-12]    ; rnd
    sub            bufq, 2*(82*73-(82*3+79))
    mov              hd, 70
    sar            maxd, 1
    mov            mind, maxd
    xor            mind, -1
.y_loop_ar1:
    mov              xq, -76
    movsx         val3d, word [bufq+xq*2-2]
.x_loop_ar1:
    movu            xm0, [bufq+xq*2-82*2-2]     ; top/left
    psrldq          xm2, xm0, 2                 ; top
    psrldq          xm1, xm0, 4                 ; top/right
    punpcklwd       xm0, xm2
    punpcklwd       xm1, xm3
    pmaddwd         xm0, xm4
    pmaddwd         xm1, xm5
    paddd           xm0, xm1
.x_loop_ar1_inner:
    movd          val0d, xm0
    psrldq          xm0, 4
    imul          val3d, cf3d
    add           val3d, val0d
    sarx          val3d, val3d, shiftd
    movsx         val0d, word [bufq+xq*2]
    add           val3d, val0d
    cmp           val3d, maxd
    cmovg         val3d, maxd
    cmp           val3d, mind
    cmovl         val3d, mind
    mov word [bufq+xq*2], val3w
    ; keep val3d in-place as left for next x iteration
    inc              xq
    jz .x_loop_ar1_end
    test             xb, 3
    jnz .x_loop_ar1_inner
    jmp .x_loop_ar1
.x_loop_ar1_end:
    add            bufq, 82*2
    dec              hd
    jg .y_loop_ar1
.ar0:
    RET

.ar2:
    DEFINE_ARGS buf, fg_data, bdmax, shift
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movq            xm0, [fg_dataq+FGData.ar_coeffs_y+5]    ; cf5-11
    vinserti128      m0, [fg_dataq+FGData.ar_coeffs_y+0], 1 ; cf0-4
    vpbroadcastw   xm10, [base+round_vals-12+shiftq*2]
    pxor             m1, m1
    punpcklwd      xm10, xm1
    pcmpgtb          m1, m0
    punpcklbw        m0, m1                                 ; cf5-11,0-4
    vpermq           m1, m0, q3333                          ; cf4
    vbroadcasti128  m11, [base+gen_shufA]
    pshufd           m6, m0, q0000                          ; cf[5,6], cf[0-1]
    vbroadcasti128  m12, [base+gen_shufB]
    pshufd           m7, m0, q1111                          ; cf[7,8], cf[2-3]
    punpckhwd       xm1, xm0
    pshufhw         xm9, xm0, q2121
    pshufd          xm8, xm1, q0000                         ; cf[4,9]
    sar          bdmaxd, 1
    punpckhqdq      xm9, xm9                                ; cf[10,11]
    movd            xm4, bdmaxd                             ; max_grain
    pcmpeqd         xm5, xm5
    sub            bufq, 2*(82*73-(82*3+79))
    pxor            xm5, xm4                                ; min_grain
    DEFINE_ARGS buf, fg_data, h, x
    mov              hd, 70
.y_loop_ar2:
    mov              xq, -76
.x_loop_ar2:
    vbroadcasti128   m2, [bufq+xq*2-82*4-4]        ; y=-2,x=[-2,+5]
    vinserti128      m1, m2, [bufq+xq*2-82*2-4], 0 ; y=-1,x=[-2,+5]
    pshufb           m0, m1, m11                   ; y=-1/-2,x=[-2/-1,-1/+0,+0/+1,+1/+2]
    pmaddwd          m0, m6
    punpckhwd       xm2, xm1                       ; y=-2/-1 interleaved, x=[+2,+5]
    pshufb           m1, m12                       ; y=-1/-2,x=[+0/+1,+1/+2,+2/+3,+3/+4]
    pmaddwd          m1, m7
    pmaddwd         xm2, xm8
    paddd            m0, m1
    vextracti128    xm1, m0, 1
    paddd           xm0, xm10
    paddd           xm2, xm0
    movu            xm0, [bufq+xq*2-4]      ; y=0,x=[-2,+5]
    paddd           xm2, xm1
    pmovsxwd        xm1, [bufq+xq*2]        ; in dwords, y=0,x=[0,3]
.x_loop_ar2_inner:
    pmaddwd         xm3, xm9, xm0
    psrldq          xm0, 2
    paddd           xm3, xm2
    psrldq          xm2, 4                  ; shift top to next pixel
    psrad           xm3, [fg_dataq+FGData.ar_coeff_shift]
    ; skip packssdw because we only care about one value
    paddd           xm3, xm1
    pminsd          xm3, xm4
    psrldq          xm1, 4
    pmaxsd          xm3, xm5
    pextrw  [bufq+xq*2], xm3, 0
    punpcklwd       xm3, xm3
    pblendw         xm0, xm3, 0010b
    inc              xq
    jz .x_loop_ar2_end
    test             xb, 3
    jnz .x_loop_ar2_inner
    jmp .x_loop_ar2
.x_loop_ar2_end:
    add            bufq, 82*2
    dec              hd
    jg .y_loop_ar2
    RET

.ar3:
    DEFINE_ARGS buf, fg_data, bdmax, shift
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    sar          bdmaxd, 1
    movq            xm7, [fg_dataq+FGData.ar_coeffs_y+ 0]    ; cf0-6
    movd            xm0, [fg_dataq+FGData.ar_coeffs_y+14]    ; cf14-16
    pinsrb          xm7, [fg_dataq+FGData.ar_coeffs_y+13], 7 ; cf0-6,13
    pinsrb          xm0, [base+pb_1], 3                      ; cf14-16,pb_1
    movd            xm1, [fg_dataq+FGData.ar_coeffs_y+21]    ; cf21-23
    vinserti128      m7, [fg_dataq+FGData.ar_coeffs_y+ 7], 1 ; cf7-13
    vinserti128      m0, [fg_dataq+FGData.ar_coeffs_y+17], 1 ; cf17-20
    vpbroadcastw   xm11, [base+round_vals+shiftq*2-12]
    movd           xm12, bdmaxd                              ; max_grain
    punpcklbw        m7, m7                                  ; sign-extension
    punpcklbw        m0, m0                                  ; sign-extension
    punpcklbw       xm1, xm1
    REPX   {psraw x, 8}, m7, m0, xm1
    pshufd           m4, m7, q0000                           ; cf[0,1] | cf[7,8]
    pshufd           m5, m7, q1111                           ; cf[2,3] | cf[9,10]
    pshufd           m6, m7, q2222                           ; cf[4,5] | cf[11,12]
    pshufd          xm7, xm7, q3333                          ; cf[6,13]
    pshufd           m8, m0, q0000                           ; cf[14,15] | cf[17,18]
    pshufd           m9, m0, q1111                           ; cf[16],pw_1 | cf[19,20]
    paddw           xm0, xm11, xm11
    pcmpeqd        xm13, xm13
    pblendw        xm10, xm1, xm0, 00001000b
    pxor           xm13, xm12                                ; min_grain
    DEFINE_ARGS buf, fg_data, h, x
    sub            bufq, 2*(82*73-(82*3+79))
    mov              hd, 70
.y_loop_ar3:
    mov              xq, -76
.x_loop_ar3:
    movu            xm0, [bufq+xq*2-82*6-6+ 0]      ; y=-3,x=[-3,+4]
    vinserti128      m0, [bufq+xq*2-82*4-6+ 0], 1   ; y=-3/-2,x=[-3,+4]
    movq            xm1, [bufq+xq*2-82*6-6+16]      ; y=-3,x=[+5,+8]
    vinserti128      m1, [bufq+xq*2-82*4-6+16], 1   ; y=-3/-2,x=[+5,+12]
    palignr          m3, m1, m0, 2                  ; y=-3/-2,x=[-2,+5]
    palignr          m1, m0, 12                     ; y=-3/-2,x=[+3,+6]
    punpckhwd        m2, m0, m3                     ; y=-3/-2,x=[+1/+2,+2/+3,+3/+4,+4/+5]
    punpcklwd        m0, m3                         ; y=-3/-2,x=[-3/-2,-2/-1,-1/+0,+0/+1]
    shufps           m3, m0, m2, q1032              ; y=-3/-2,x=[-1/+0,+0/+1,+1/+2,+2/+3]
    pmaddwd          m0, m4
    pmaddwd          m2, m6
    pmaddwd          m3, m5
    paddd            m0, m2
    movu            xm2, [bufq+xq*2-82*2-6+ 0]      ; y=-1,x=[-3,+4]
    vinserti128      m2, [bufq+xq*2-82*2-6+ 6], 1   ; y=-1,x=[+1,+8]
    paddd            m0, m3
    psrldq           m3, m2, 2
    punpcklwd        m3, m2, m3                     ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1]
    pmaddwd          m3, m8                         ;      x=[+0/+1,+1/+2,+2/+3,+3/+4]
    paddd            m0, m3
    psrldq           m3, m2, 4
    psrldq           m2, 6
    vpblendd         m2, m11, 0x0f                  ; rounding constant
    punpcklwd        m3, m2                         ; y=-1,x=[-1/rnd,+0/rnd,+1/rnd,+2/rnd]
    pmaddwd          m3, m9                         ;      x=[+2/+3,+3/+4,+4/+5,+5,+6]
    vextracti128    xm2, m1, 1
    punpcklwd       xm1, xm2
    pmaddwd         xm1, xm7                        ; y=-3/-2 interleaved,x=[+3,+4,+5,+6]
    paddd            m0, m3
    vextracti128    xm2, m0, 1
    paddd           xm0, xm1
    movu            xm1, [bufq+xq*2-6]        ; y=0,x=[-3,+4]
    paddd           xm0, xm2
.x_loop_ar3_inner:
    pmaddwd         xm2, xm1, xm10
    pshuflw         xm3, xm2, q1032
    paddd           xm2, xm0                ; add top
    paddd           xm2, xm3                ; left+cur
    psrldq          xm0, 4
    psrad           xm2, [fg_dataq+FGData.ar_coeff_shift]
    ; skip packssdw because we only care about one value
    pminsd          xm2, xm12
    pmaxsd          xm2, xm13
    pextrw  [bufq+xq*2], xm2, 0
    pslldq          xm2, 4
    psrldq          xm1, 2
    pblendw         xm1, xm2, 0100b
    inc              xq
    jz .x_loop_ar3_end
    test             xb, 3
    jnz .x_loop_ar3_inner
    jmp .x_loop_ar3
.x_loop_ar3_end:
    add            bufq, 82*2
    dec              hd
    jg .y_loop_ar3
    RET

%macro GEN_GRAIN_UV_FN 3 ; ss_name, ss_x, ss_y
INIT_XMM avx2
cglobal generate_grain_uv_%1_16bpc, 4, 11, 8, buf, bufy, fg_data, uv, bdmax
%define base r8-generate_grain_uv_%1_16bpc_avx2_table
    lea              r8, [generate_grain_uv_%1_16bpc_avx2_table]
    movifnidn    bdmaxd, bdmaxm
    vpbroadcastw    xm0, [fg_dataq+FGData.seed]
    mov             r5d, [fg_dataq+FGData.grain_scale_shift]
    movq            xm1, [base+next_upperbit_mask]
    lea             r6d, [bdmaxq+1]
    movq            xm4, [base+mul_bits]
    shr             r6d, 11             ; 0 for 10bpc, 2 for 12bpc
    movq            xm5, [base+hmul_bits]
    sub              r5, r6
    mova            xm6, [base+pb_mask]
    vpbroadcastd    xm2, [base+pw_seed_xor+uvq*4]
    vpbroadcastw    xm7, [base+round+r5*2-2]
    pxor            xm0, xm2
    lea              r6, [gaussian_sequence]
%if %2
    mov             r7d, 73-35*%3
    add            bufq, 44*2
.loop_y:
    mov              r5, -44*2
%else
    mov              r5, -82*73*2
    sub            bufq, r5
%endif
.loop_x:
    pand            xm2, xm0, xm1
    psrlw           xm3, xm2, 10
    por             xm2, xm3            ; bits 0xf, 0x1e, 0x3c and 0x78 are set
    pmullw          xm2, xm4            ; bits 0x0f00 are set
    pmulhuw         xm0, xm5
    pshufb          xm3, xm6, xm2       ; set 15th bit for next 4 seeds
    psllq           xm2, xm3, 30
    por             xm2, xm3
    psllq           xm3, xm2, 15
    por             xm2, xm0            ; aggregate each bit into next seed's high bit
    por             xm2, xm3            ; 4 next output seeds
    pshuflw         xm0, xm2, q3333
    psrlw           xm2, 5
    movq            r10, xm2
    movzx           r9d, r10w
    movd            xm2, [r6+r9*2]
    rorx             r9, r10, 32
    shr            r10d, 16
    pinsrw          xm2, [r6+r10*2], 1
    movzx          r10d, r9w
    pinsrw          xm2, [r6+r10*2], 2
    shr             r9d, 16
    pinsrw          xm2, [r6+r9*2], 3
    paddw           xm2, xm2            ; otherwise bpc=12 w/ grain_scale_shift=0
    pmulhrsw        xm2, xm7            ; shifts by 0, which pmulhrsw does not support
    movq      [bufq+r5], xm2
    add              r5, 8
    jl .loop_x
%if %2
    add            bufq, 82*2
    dec             r7d
    jg .loop_y
%endif

    ; auto-regression code
    movsxd           r6, [fg_dataq+FGData.ar_coeff_lag]
    movsxd           r6, [r8+r6*4]
    add              r6, r8
    jmp              r6

INIT_YMM avx2
.ar0:
    DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift
    imul            uvd, 28
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    vpbroadcastb     m0, [fg_dataq+FGData.ar_coeffs_uv+uvq]
    sar          bdmaxd, 1
    vpbroadcastd     m4, [base+gen_ar0_shift-24+shiftq*4]
    movd            xm6, bdmaxd
    pcmpeqw          m7, m7
    pmaddubsw        m4, m0  ; ar_coeff << (14 - shift)
    vpbroadcastw     m6, xm6 ; max_gain
    pxor             m7, m6  ; min_grain
    DEFINE_ARGS buf, bufy, h, x
%if %2
    vpbroadcastw     m5, [base+hmul_bits+2+%3*2]
    sub            bufq, 2*(82*(73-35*%3)+82-(82*3+41))
%else
    sub            bufq, 2*(82*70-3)
%endif
    add           bufyq, 2*(3+82*3)
    mov              hd, 70-35*%3
.y_loop_ar0:
%if %2
    ; first 32 pixels
    movu            xm0, [bufyq+16*0]
    vinserti128      m0, [bufyq+16*2], 1
    movu            xm1, [bufyq+16*1]
    vinserti128      m1, [bufyq+16*3], 1
%if %3
    movu            xm2, [bufyq+82*2+16*0]
    vinserti128      m2, [bufyq+82*2+16*2], 1
    movu            xm3, [bufyq+82*2+16*1]
    vinserti128      m3, [bufyq+82*2+16*3], 1
    paddw            m0, m2
    paddw            m1, m3
%endif
    phaddw           m0, m1
    movu            xm1, [bufyq+16*4]
    vinserti128      m1, [bufyq+16*6], 1
    movu            xm2, [bufyq+16*5]
    vinserti128      m2, [bufyq+16*7], 1
%if %3
    movu            xm3, [bufyq+82*2+16*4]
    vinserti128      m3, [bufyq+82*2+16*6], 1
    paddw            m1, m3
    movu            xm3, [bufyq+82*2+16*5]
    vinserti128      m3, [bufyq+82*2+16*7], 1
    paddw            m2, m3
%endif
    phaddw           m1, m2
    pmulhrsw         m0, m5
    pmulhrsw         m1, m5
%else
    xor              xd, xd
.x_loop_ar0:
    movu             m0, [bufyq+xq*2]
    movu             m1, [bufyq+xq*2+32]
%endif
    paddw            m0, m0
    paddw            m1, m1
    pmulhrsw         m0, m4
    pmulhrsw         m1, m4
%if %2
    paddw            m0, [bufq+ 0]
    paddw            m1, [bufq+32]
%else
    paddw            m0, [bufq+xq*2+ 0]
    paddw            m1, [bufq+xq*2+32]
%endif
    pminsw           m0, m6
    pminsw           m1, m6
    pmaxsw           m0, m7
    pmaxsw           m1, m7
%if %2
    movu      [bufq+ 0], m0
    movu      [bufq+32], m1

    ; last 6 pixels
    movu            xm0, [bufyq+32*4]
    movu            xm1, [bufyq+32*4+16]
%if %3
    paddw           xm0, [bufyq+32*4+82*2]
    paddw           xm1, [bufyq+32*4+82*2+16]
%endif
    phaddw          xm0, xm1
    movu            xm1, [bufq+32*2]
    pmulhrsw        xm0, xm5
    paddw           xm0, xm0
    pmulhrsw        xm0, xm4
    paddw           xm0, xm1
    pminsw          xm0, xm6
    pmaxsw          xm0, xm7
    vpblendd        xm0, xm1, 0x08
    movu    [bufq+32*2], xm0
%else
    movu [bufq+xq*2+ 0], m0
    movu [bufq+xq*2+32], m1
    add              xd, 32
    cmp              xd, 64
    jl .x_loop_ar0

    ; last 12 pixels
    movu             m0, [bufyq+64*2]
    movu             m1, [bufq+64*2]
    paddw            m0, m0
    pmulhrsw         m0, m4
    paddw            m0, m1
    pminsw           m0, m6
    pmaxsw           m0, m7
    vpblendd         m0, m1, 0xc0
    movu    [bufq+64*2], m0
%endif
    add            bufq, 82*2
    add           bufyq, 82*2<<%3
    dec              hd
    jg .y_loop_ar0
    RET

INIT_XMM avx2
.ar1:
    DEFINE_ARGS buf, bufy, fg_data, uv, max, cf3, min, val3, x, shift
    imul            uvd, 28
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movsx          cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3]
    movd            xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq]
    pinsrb          xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3
    DEFINE_ARGS buf, bufy, h, val0, max, cf3, min, val3, x, shift
    pmovsxbw        xm4, xm4
    pshufd          xm5, xm4, q1111
    pshufd          xm4, xm4, q0000
    pmovsxwd        xm3, [base+round_vals+shiftq*2-12]    ; rnd
    vpbroadcastw    xm6, [base+hmul_bits+2+%3*2]
    vpbroadcastd    xm3, xm3
%if %2
    sub            bufq, 2*(82*(73-35*%3)+44-(82*3+41))
%else
    sub            bufq, 2*(82*69+3)
%endif
    add           bufyq, 2*(79+82*3)
    mov              hd, 70-35*%3
    sar            maxd, 1
    mov            mind, maxd
    xor            mind, -1
.y_loop_ar1:
    mov              xq, -(76>>%2)
    movsx         val3d, word [bufq+xq*2-2]
.x_loop_ar1:
    movu            xm0, [bufq+xq*2-82*2-2] ; top/left
%if %2
    movu            xm2, [bufyq+xq*4]
%else
    movq            xm2, [bufyq+xq*2]
%endif
%if %2
%if %3
    phaddw          xm2, [bufyq+xq*4+82*2]
    punpckhqdq      xm1, xm2, xm2
    paddw           xm2, xm1
%else
    phaddw          xm2, xm2
%endif
    pmulhrsw        xm2, xm6
%endif
    psrldq          xm1, xm0, 4             ; top/right
    punpcklwd       xm1, xm2
    psrldq          xm2, xm0, 2             ; top
    punpcklwd       xm0, xm2
    pmaddwd         xm1, xm5
    pmaddwd         xm0, xm4
    paddd           xm1, xm3
    paddd           xm0, xm1
.x_loop_ar1_inner:
    movd          val0d, xm0
    psrldq          xm0, 4
    imul          val3d, cf3d
    add           val3d, val0d
    sarx          val3d, val3d, shiftd
    movsx         val0d, word [bufq+xq*2]
    add           val3d, val0d
    cmp           val3d, maxd
    cmovg         val3d, maxd
    cmp           val3d, mind
    cmovl         val3d, mind
    mov word [bufq+xq*2], val3w
    ; keep val3d in-place as left for next x iteration
    inc              xq
    jz .x_loop_ar1_end
    test             xb, 3
    jnz .x_loop_ar1_inner
    jmp .x_loop_ar1
.x_loop_ar1_end:
    add            bufq, 82*2
    add           bufyq, 82*2<<%3
    dec              hd
    jg .y_loop_ar1
    RET

INIT_YMM avx2
.ar2:
%if WIN64
    ; xmm6 and xmm7 already saved
    %assign xmm_regs_used 13 + %2
    %assign stack_size_padded 136
    SUB             rsp, stack_size_padded
    movaps   [rsp+16*2], xmm8
    movaps   [rsp+16*3], xmm9
    movaps   [rsp+16*4], xmm10
    movaps   [rsp+16*5], xmm11
    movaps   [rsp+16*6], xmm12
%if %2
    movaps   [rsp+16*7], xmm13
%endif
%endif
    DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    imul            uvd, 28
    vbroadcasti128  m10, [base+gen_shufA]
    sar          bdmaxd, 1
    vbroadcasti128  m11, [base+gen_shufB]
    movd            xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 5]
    pinsrb          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+12], 4
    pinsrb          xm7, [base+pb_1], 5
    pinsrw          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+10], 3
    movhps          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0]
    pinsrb          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 9], 13
    pmovsxbw         m7, xm7
    movd            xm8, bdmaxd             ; max_grain
    pshufd           m4, m7, q0000
    vpbroadcastw   xm12, [base+round_vals-12+shiftq*2]
    pshufd           m5, m7, q1111
    pcmpeqd         xm9, xm9
    pshufd           m6, m7, q2222
    pxor            xm9, xm8                ; min_grain
    pshufd          xm7, xm7, q3333
    DEFINE_ARGS buf, bufy, fg_data, h, x
%if %2
    vpbroadcastw   xm13, [base+hmul_bits+2+%3*2]
    sub            bufq, 2*(82*(73-35*%3)+44-(82*3+41))
%else
    sub            bufq, 2*(82*69+3)
%endif
    add           bufyq, 2*(79+82*3)
    mov              hd, 70-35*%3
.y_loop_ar2:
    mov              xq, -(76>>%2)
.x_loop_ar2:
    vbroadcasti128   m3, [bufq+xq*2-82*2-4]        ; y=-1,x=[-2,+5]
    vinserti128      m2, m3, [bufq+xq*2-82*4-4], 1 ; y=-2,x=[-2,+5]
    pshufb           m0, m2, m10                   ; y=-1/-2,x=[-2/-1,-1/+0,+0/+1,+1/+2]
    pmaddwd          m0, m4
    pshufb           m1, m2, m11                   ; y=-1/-2,x=[+0/+1,+1/+2,+2/+3,+3/+4]
    pmaddwd          m1, m5
    punpckhwd        m2, m3                        ; y=-2/-1 interleaved, x=[+2,+5]
%if %2
    movu            xm3, [bufyq+xq*4]
%if %3
    paddw           xm3, [bufyq+xq*4+82*2]
%endif
    phaddw          xm3, xm3
    pmulhrsw        xm3, xm13
%else
    movq            xm3, [bufyq+xq*2]
%endif
    punpcklwd       xm3, xm12                   ; luma, round interleaved
    vpblendd         m2, m3, 0x0f
    pmaddwd          m2, m6
    paddd            m1, m0
    movu            xm0, [bufq+xq*2-4]      ; y=0,x=[-2,+5]
    paddd            m2, m1
    vextracti128    xm1, m2, 1
    paddd           xm2, xm1
    pshufd          xm1, xm0, q3321
    pmovsxwd        xm1, xm1                ; y=0,x=[0,3] in dword
.x_loop_ar2_inner:
    pmaddwd         xm3, xm7, xm0
    paddd           xm3, xm2
    psrldq          xm2, 4                  ; shift top to next pixel
    psrad           xm3, [fg_dataq+FGData.ar_coeff_shift]
    ; we do not need to packssdw since we only care about one value
    paddd           xm3, xm1
    psrldq          xm1, 4
    pminsd          xm3, xm8
    pmaxsd          xm3, xm9
    pextrw  [bufq+xq*2], xm3, 0
    psrldq          xm0, 2
    pslldq          xm3, 2
    pblendw         xm0, xm3, 00000010b
    inc              xq
    jz .x_loop_ar2_end
    test             xb, 3
    jnz .x_loop_ar2_inner
    jmp .x_loop_ar2
.x_loop_ar2_end:
    add            bufq, 82*2
    add           bufyq, 82*2<<%3
    dec              hd
    jg .y_loop_ar2
    RET

.ar3:
%if WIN64
    ; xmm6 and xmm7 already saved
    %assign stack_offset 32
    %assign xmm_regs_used 14 + %2
    %assign stack_size_padded 152
    SUB             rsp, stack_size_padded
    movaps   [rsp+16*2], xmm8
    movaps   [rsp+16*3], xmm9
    movaps   [rsp+16*4], xmm10
    movaps   [rsp+16*5], xmm11
    movaps   [rsp+16*6], xmm12
    movaps   [rsp+16*7], xmm13
%if %2
    movaps   [rsp+16*8], xmm14
%endif
%endif
    DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    imul            uvd, 28
    vpbroadcastw   xm11, [base+round_vals-12+shiftq*2]
    sar          bdmaxd, 1
    movq            xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0]
    pinsrb          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+24], 7 ; luma
    movhps          xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 7]
    pmovsxbw         m7, xm7
%if %2
    vpbroadcastw   xm14, [base+hmul_bits+2+%3*2]
%endif
    pshufd           m4, m7, q0000
    pshufd           m5, m7, q1111
    pshufd           m6, m7, q2222
    pshufd           m7, m7, q3333
    movd            xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+14]
    pinsrb          xm0, [base+pb_1], 3
    pinsrd          xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+21], 1
    pinsrd          xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+17], 2
    pmovsxbw         m0, xm0
    movd           xm12, bdmaxd                 ; max_grain
    pshufd           m8, m0, q0000
    pshufd           m9, m0, q1111
    pcmpeqd        xm13, xm13
    punpckhqdq     xm10, xm0, xm0
    pxor           xm13, xm12                   ; min_grain
    pinsrw         xm10, [base+round_vals-10+shiftq*2], 3
    DEFINE_ARGS buf, bufy, fg_data, h, unused, x
%if %2
    sub            bufq, 2*(82*(73-35*%3)+44-(82*3+41))
%else
    sub            bufq, 2*(82*69+3)
%endif
    add           bufyq, 2*(79+82*3)
    mov              hd, 70-35*%3
.y_loop_ar3:
    mov              xq, -(76>>%2)
.x_loop_ar3:
    movu            xm2, [bufq+xq*2-82*6-6+ 0]    ; y=-3,x=[-3,+4]
    vinserti128      m2, [bufq+xq*2-82*4-6+ 0], 1 ; y=-3/-2,x=[-3,+4]
    movq            xm1, [bufq+xq*2-82*6-6+16]    ; y=-3,x=[+5,+8]
    vinserti128      m1, [bufq+xq*2-82*4-6+16], 1 ; y=-3/-2,x=[+5,+12]
    palignr          m3, m1, m2, 2                ; y=-3/-2,x=[-2,+5]
    palignr          m1, m2, 12                   ; y=-3/-2,x=[+3,+6]
    punpcklwd        m0, m2, m3                   ; y=-3/-2,x=[-3/-2,-2/-1,-1/+0,+0/+1]
    punpckhwd        m2, m3                       ; y=-3/-2,x=[+1/+2,+2/+3,+3/+4,+4/+5]
    shufps           m3, m0, m2, q1032            ; y=-3/-2,x=[-1/+0,+0/+1,+1/+2,+2/+3]
    pmaddwd          m0, m4
    pmaddwd          m2, m6
    pmaddwd          m3, m5
    paddd            m0, m2
    paddd            m0, m3
    movu            xm2, [bufq+xq*2-82*2-6+ 0]    ; y=-1,x=[-3,+4]
    vinserti128      m2, [bufq+xq*2-82*2-6+ 6], 1 ; y=-1,x=[+1,+8]
%if %2
    movu            xm3, [bufyq+xq*4]
%if %3
    paddw           xm3, [bufyq+xq*4+82*2]
%endif
    phaddw          xm3, xm3
    pmulhrsw        xm3, xm14
%else
    movq            xm3, [bufyq+xq*2]
%endif
    punpcklwd        m1, m3
    pmaddwd          m1, m7
    paddd            m0, m1
    psrldq           m1, m2, 4
    psrldq           m3, m2, 6
    vpblendd         m3, m11, 0x0f                ; rounding constant
    punpcklwd        m1, m3                       ; y=-1,x=[-1/rnd,+0/rnd,+1/rnd,+2/rnd]
    pmaddwd          m1, m9                       ;      x=[+2/+3,+3/+4,+4/+5,+5,+6]
    psrldq           m3, m2, 2
    punpcklwd        m2, m3                       ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1]
    pmaddwd          m2, m8                       ;      x=[+0/+1,+1/+2,+2/+3,+3/+4]
    paddd            m0, m1
    movu            xm1, [bufq+xq*2-6]            ; y=0,x=[-3,+4]
    paddd            m0, m2
    vextracti128    xm2, m0, 1
    paddd           xm0, xm2
.x_loop_ar3_inner:
    pmaddwd         xm2, xm1, xm10
    pshuflw         xm3, xm2, q1032
    paddd           xm2, xm0                      ; add top
    paddd           xm2, xm3                      ; left+cur
    psrldq          xm0, 4
    psrad           xm2, [fg_dataq+FGData.ar_coeff_shift]
    psrldq          xm1, 2
    ; no need to packssdw since we only care about one value
    pminsd          xm2, xm12
    pmaxsd          xm2, xm13
    pextrw  [bufq+xq*2], xm2, 0
    pslldq          xm2, 4
    pblendw         xm1, xm2, 00000100b
    inc              xq
    jz .x_loop_ar3_end
    test             xb, 3
    jnz .x_loop_ar3_inner
    jmp .x_loop_ar3
.x_loop_ar3_end:
    add            bufq, 82*2
    add           bufyq, 82*2<<%3
    dec              hd
    jg .y_loop_ar3
    RET
%endmacro

cglobal fgy_32x32xn_16bpc, 6, 14, 16, dst, src, stride, fg_data, w, scaling, \
                                      grain_lut, unused, sby, see
%define base r11-grain_min
    lea             r11, [grain_min]
    mov             r6d, r9m ; bdmax
    mov             r9d, [fg_dataq+FGData.clip_to_restricted_range]
    mov             r7d, [fg_dataq+FGData.scaling_shift]
    mov            sbyd, sbym
    vpbroadcastd     m8, r9m
    shr             r6d, 11  ; is_12bpc
    vpbroadcastd     m9, [base+grain_min+r6*4]
    shlx           r10d, r9d, r6d
    vpbroadcastd    m10, [base+grain_max+r6*4]
    lea             r9d, [r6+r9*4]
    vpbroadcastw    m11, [base+mul_bits+r7*2-12]
    vpbroadcastd    m12, [base+fg_min+r10*4]
    vpbroadcastd    m13, [base+fg_max+r9*4]
    test           sbyd, sbyd
    setnz           r7b
    vpbroadcastd    m14, [base+pd_16]
    test            r7b, [fg_dataq+FGData.overlap_flag]
    jnz .vertical_overlap

    imul           seed, sbyd, (173 << 24) | 37
    add            seed, (105 << 24) | 178
    rorx           seed, seed, 24
    movzx          seed, seew
    xor            seed, [fg_dataq+FGData.seed]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, src_bak

    lea        src_bakq, [srcq+wq*2]
    neg              wq
    sub            dstq, srcq

.loop_x:
    rorx             r6, seeq, 1
    or             seed, 0xEFF4
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d                ; updated seed
    rorx          offyd, seed, 8
    rorx          offxq, seeq, 12
    and           offyd, 0xf
    imul          offyd, 164
    lea           offyd, [offyq+offxq*2+747] ; offy*stride+offx

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, src_bak

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
.loop_y:
    ; scaling[src]
    mova             m0, [srcq+ 0]
    mova             m1, [srcq+32]
    pand             m4, m8, m0
    psrld            m3, m0, 16
    mova             m6, m9
    vpgatherdd       m2, [scalingq+m4-0], m9
    pand             m3, m8
    mova             m9, m6
    vpgatherdd       m4, [scalingq+m3-2], m6
    pand             m5, m8, m1
    mova             m6, m9
    vpgatherdd       m3, [scalingq+m5-0], m9
    pblendw          m4, m2, 0x55
    psrld            m2, m1, 16
    mova             m9, m6
    pand             m2, m8
    vpgatherdd       m5, [scalingq+m2-2], m6
    pblendw          m5, m3, 0x55

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m4, [grain_lutq+offxyq*2]
    pmulhrsw         m5, [grain_lutq+offxyq*2+32]

    ; dst = clip_pixel(src, noise)
    paddw            m0, m4
    paddw            m1, m5
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova [dstq+srcq+ 0], m0
    mova [dstq+srcq+32], m1

    add            srcq, strideq
    add      grain_lutq, 82*2
    dec              hd
    jg .loop_y
    add              wq, 32
    jge .end
    lea            srcq, [src_bakq+wq*2]
    cmp byte [fg_dataq+FGData.overlap_flag], 0
    je .loop_x
    movq            xm7, [pw_27_17_17_27]
    cmp       dword r8m, 0 ; sby
    jne .loop_x_hv_overlap

    ; horizontal overlap (without vertical overlap)
.loop_x_h_overlap:
    rorx             r6, seeq, 1
    or             seed, 0xEFF4
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d                ; updated seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, src_bak, left_offxy

    lea     left_offxyd, [offyq+32]         ; previous column's offy*stride+offx
    rorx          offyd, seed, 8
    rorx          offxq, seeq, 12
    and           offyd, 0xf
    imul          offyd, 164
    lea           offyd, [offyq+offxq*2+747] ; offy*stride+offx

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, src_bak, left_offxy

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
.loop_y_h_overlap:
    ; scaling[src]
    mova             m0, [srcq+ 0]
    mova             m1, [srcq+32]
    pand             m4, m8, m0
    psrld            m3, m0, 16
    mova             m6, m9
    vpgatherdd       m2, [scalingq+m4-0], m9
    pand             m3, m8
    mova             m9, m6
    vpgatherdd       m4, [scalingq+m3-2], m6
    pand             m5, m8, m1
    mova             m6, m9
    vpgatherdd       m3, [scalingq+m5-0], m9
    pblendw          m4, m2, 0x55
    psrld            m2, m1, 16
    mova             m9, m6
    pand             m2, m8
    vpgatherdd       m5, [scalingq+m2-2], m6
    pblendw          m5, m3, 0x55

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq*2]
    movd            xm6, [grain_lutq+left_offxyq*2]
    punpcklwd       xm6, xm3
    pmaddwd         xm6, xm7
    paddd           xm6, xm14
    psrad           xm6, 5
    packssdw        xm6, xm6
    pmaxsw          xm6, xm9
    pminsw          xm6, xm10
    vpblendd         m3, m6, 0x01

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m4, m3
    pmulhrsw         m5, [grain_lutq+offxyq*2+32]

    ; dst = clip_pixel(src, noise)
    paddw            m0, m4
    paddw            m1, m5
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova [dstq+srcq+ 0], m0
    mova [dstq+srcq+32], m1

    add            srcq, strideq
    add      grain_lutq, 82*2
    dec              hd
    jg .loop_y_h_overlap
    add              wq, 32
    jge .end
    lea            srcq, [src_bakq+wq*2]
    cmp       dword r8m, 0 ; sby
    jne .loop_x_hv_overlap
    jmp .loop_x_h_overlap

.vertical_overlap:
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \
                sby, see, src_bak

    movzx          sbyd, sbyb
    imul           seed, [fg_dataq+FGData.seed], 0x00010001
    imul            r7d, sbyd, 173 * 0x00010001
    imul           sbyd, 37 * 0x01000100
    add             r7d, (105 << 16) | 188
    add            sbyd, (178 << 24) | (141 << 8)
    and             r7d, 0x00ff00ff
    and            sbyd, 0xff00ff00
    xor            seed, r7d
    xor            seed, sbyd               ; (cur_seed << 16) | top_seed

    lea        src_bakq, [srcq+wq*2]
    neg              wq
    sub            dstq, srcq

.loop_x_v_overlap:
    vpbroadcastd    m15, [pw_27_17_17_27]

    ; we assume from the block above that bits 8-15 of r7d are zero'ed
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp            r7b                     ; parity of top_seed
    shr            seed, 16
    shl             r7d, 16
    test           seeb, seeh
    setp            r7b                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor             r7d, r6d
    rorx           seed, r7d, 1             ; updated (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, src_bak, unused, top_offxy

    rorx          offyd, seed, 8
    rorx          offxd, seed, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyd, [offyq+offxq*2+0x10001*747+32*82]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, src_bak, unused, top_offxy

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
    movzx    top_offxyd, offxyw
    shr          offxyd, 16
.loop_y_v_overlap:
    ; scaling[src]
    mova             m0, [srcq+ 0]
    mova             m1, [srcq+32]
    pand             m4, m8, m0
    psrld            m3, m0, 16
    mova             m6, m9
    vpgatherdd       m2, [scalingq+m4-0], m9
    pand             m3, m8
    mova             m9, m6
    vpgatherdd       m4, [scalingq+m3-2], m6
    pand             m5, m8, m1
    mova             m6, m9
    vpgatherdd       m3, [scalingq+m5-0], m9
    pblendw          m2, m4, 0xaa
    psrld            m4, m1, 16
    mova             m9, m6
    pand             m4, m8
    vpgatherdd       m5, [scalingq+m4-2], m6
    pblendw          m3, m5, 0xaa

    ; grain = grain_lut[offy+y][offx+x]
    movu             m6, [grain_lutq+offxyq*2]
    movu             m5, [grain_lutq+top_offxyq*2]
    punpcklwd        m4, m5, m6
    punpckhwd        m5, m6
    pmaddwd          m4, m15
    pmaddwd          m5, m15
    movu             m7, [grain_lutq+offxyq*2+32]
    movu             m6, [grain_lutq+top_offxyq*2+32]
    paddd            m4, m14
    paddd            m5, m14
    psrad            m4, 5
    psrad            m5, 5
    packssdw         m4, m5
    punpcklwd        m5, m6, m7
    punpckhwd        m6, m7
    pmaddwd          m5, m15
    pmaddwd          m6, m15
    paddd            m5, m14
    paddd            m6, m14
    psrad            m5, 5
    psrad            m6, 5
    packssdw         m5, m6
    pmaxsw           m4, m9
    pmaxsw           m5, m9
    pminsw           m4, m10
    pminsw           m5, m10

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmaddubsw        m2, m11
    pmaddubsw        m3, m11
    paddw            m2, m2
    paddw            m3, m3
    pmulhrsw         m4, m2
    pmulhrsw         m5, m3

    ; dst = clip_pixel(src, noise)
    paddw            m0, m4
    paddw            m1, m5
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova [dstq+srcq+ 0], m0
    mova [dstq+srcq+32], m1

    add            srcq, strideq
    add      grain_lutq, 82*2
    dec              hb
    jz .end_y_v_overlap
    vpbroadcastd    m15, [pw_27_17_17_27+4] ; swap weights for second v-overlap line
    ; 2 lines get vertical overlap, then fall back to non-overlap code for
    ; remaining (up to) 30 lines
    add              hd, 0x80000000
    jnc .loop_y_v_overlap
    jmp .loop_y
.end_y_v_overlap:
    add              wq, 32
    jge .end
    lea            srcq, [src_bakq+wq*2]

    ; since fg_dataq.overlap is guaranteed to be set, we never jump
    ; back to .loop_x_v_overlap, and instead always fall-through to
    ; h+v overlap

.loop_x_hv_overlap:
    vpbroadcastd    m15, [pw_27_17_17_27]

    ; we assume from the block above that bits 8-15 of r7d are zero'ed
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp            r7b                     ; parity of top_seed
    shr            seed, 16
    shl             r7d, 16
    test           seeb, seeh
    setp            r7b                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor             r7d, r6d
    rorx           seed, r7d, 1             ; updated (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy

    lea  topleft_offxyd, [top_offxyq+32]
    lea     left_offxyd, [offyq+32]
    rorx          offyd, seed, 8
    rorx          offxd, seed, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyd, [offyq+offxq*2+0x10001*747+32*82]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, src_bak, left_offxy, top_offxy, topleft_offxy

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
    movzx    top_offxyd, offxyw
    shr          offxyd, 16
.loop_y_hv_overlap:
    ; scaling[src]
    mova             m0, [srcq+ 0]
    mova             m1, [srcq+32]
    pand             m4, m8, m0
    psrld            m3, m0, 16
    mova             m6, m9
    vpgatherdd       m2, [scalingq+m4-0], m9
    pand             m3, m8
    mova             m9, m6
    vpgatherdd       m4, [scalingq+m3-2], m6
    pand             m5, m8, m1
    mova             m6, m9
    vpgatherdd       m3, [scalingq+m5-0], m9
    pblendw          m2, m4, 0xaa
    psrld            m4, m1, 16
    mova             m9, m6
    pand             m4, m8
    vpgatherdd       m5, [scalingq+m4-2], m6
    pblendw          m3, m5, 0xaa

    ; grain = grain_lut[offy+y][offx+x]
    movu             m7, [grain_lutq+offxyq*2]
    movd            xm6, [grain_lutq+left_offxyq*2]
    movu             m5, [grain_lutq+top_offxyq*2]
    movd            xm4, [grain_lutq+topleft_offxyq*2]
    ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
    punpcklwd       xm6, xm7
    punpcklwd       xm4, xm5
    punpcklqdq      xm6, xm4
    movddup         xm4, [pw_27_17_17_27]
    pmaddwd         xm6, xm4
    paddd           xm6, xm14
    psrad           xm6, 5
    packssdw        xm6, xm6
    pmaxsw          xm6, xm9
    pminsw          xm6, xm10
    pshuflw         xm4, xm6, q1032
    vpblendd         m6, m7, 0xfe
    vpblendd         m4, m5, 0xfe
    ; followed by v interpolation (top | cur -> cur)
    punpckhwd        m5, m7
    pmaddwd          m5, m15
    punpcklwd        m4, m6
    pmaddwd          m4, m15
    movu             m7, [grain_lutq+offxyq*2+32]
    movu             m6, [grain_lutq+top_offxyq*2+32]
    paddd            m5, m14
    paddd            m4, m14
    psrad            m5, 5
    psrad            m4, 5
    packssdw         m4, m5
    punpcklwd        m5, m6, m7
    punpckhwd        m6, m7
    pmaddwd          m5, m15
    pmaddwd          m6, m15
    paddd            m5, m14
    paddd            m6, m14
    psrad            m5, 5
    psrad            m6, 5
    packssdw         m5, m6
    pmaxsw           m4, m9
    pmaxsw           m5, m9
    pminsw           m4, m10
    pminsw           m5, m10

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmaddubsw        m2, m11
    pmaddubsw        m3, m11
    paddw            m2, m2
    paddw            m3, m3
    pmulhrsw         m4, m2
    pmulhrsw         m5, m3

    ; dst = clip_pixel(src, noise)
    paddw            m0, m4
    paddw            m1, m5
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova [dstq+srcq+ 0], m0
    mova [dstq+srcq+32], m1

    add            srcq, strideq
    add      grain_lutq, 82*2
    dec              hb
    jz .end_y_hv_overlap
    vpbroadcastd    m15, [pw_27_17_17_27+4] ; swap weights for second v-overlap line
    ; 2 lines get vertical overlap, then fall back to non-overlap code for
    ; remaining (up to) 30 lines
    add              hd, 0x80000000
    jnc .loop_y_hv_overlap
    movq            xm7, [pw_27_17_17_27]
    jmp .loop_y_h_overlap
.end_y_hv_overlap:
    add              wq, 32
    lea            srcq, [src_bakq+wq*2]
    jl .loop_x_hv_overlap
.end:
    RET

%macro FGUV_FN 3 ; name, ss_hor, ss_ver
cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \
                                           grain_lut, h, sby, luma, lstride, uv_pl, is_id
%define base r12-grain_min
    lea             r12, [grain_min]
    mov             r9d, r13m               ; bdmax
    mov             r7d, [fg_dataq+FGData.scaling_shift]
    mov            r11d, is_idm
    mov            sbyd, sbym
    vpbroadcastw    m11, [base+mul_bits+r7*2-12]
    mov             r6d, [fg_dataq+FGData.clip_to_restricted_range]
    shr             r9d, 11                 ; is_12bpc
    vpbroadcastd     m8, [base+grain_min+r9*4]
    shlx           r10d, r6d, r9d
    vpbroadcastd     m9, [base+grain_max+r9*4]
    vpbroadcastw    m10, r13m
    shlx            r6d, r6d, r11d
    vpbroadcastd    m12, [base+fg_min+r10*4]
    lea             r6d, [r9+r6*2]
    vpbroadcastd    m13, [base+fg_max+r6*4]
    test           sbyd, sbyd
    setnz           r7b
    cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
    jne .csfl

%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                unused, sby, see, overlap

%if %1
    mov             r6d, r11m
    vpbroadcastd     m0, [base+pb_8_9_0_1]
    vpbroadcastd     m1, [base+uv_offset_mul+r9*4]
    vbroadcasti128  m14, [fg_dataq+FGData.uv_mult+r6*4]
    vpbroadcastd    m15, [fg_dataq+FGData.uv_offset+r6*4]
    pshufb          m14, m0 ; { uv_luma_mult, uv_mult }
    pmaddwd         m15, m1
%else
%if %2
    vpbroadcastq    m15, [base+pw_23_22]
%else
    vpbroadcastq    m15, [base+pw_27_17_17_27]
%endif
    vpbroadcastd    m14, [base+pd_16]
%endif
    test            r7b, [fg_dataq+FGData.overlap_flag]
    jnz %%vertical_overlap

    imul           seed, sbyd, (173 << 24) | 37
    add            seed, (105 << 24) | 178
    rorx           seed, seed, 24
    movzx          seed, seew
    xor            seed, [fg_dataq+FGData.seed]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                unused2, unused3, see, unused4, unused5, unused6, luma, lstride

    mov           lumaq, r9mp
    mov        lstrideq, r10mp
    lea             r10, [srcq+wq*2]
    lea             r11, [dstq+wq*2]
    lea             r12, [lumaq+wq*(2<<%2)]
    mov            r9mp, r10
    mov           r11mp, r11
    mov           r12mp, r12
    neg              wq

%%loop_x:
    rorx             r6, seeq, 1
    or             seed, 0xEFF4
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d               ; updated seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, unused1, unused2, unused3, luma, lstride

    rorx          offyd, seed, 8
    rorx          offxq, seeq, 12
    and           offyd, 0xf
    imul          offyd, 164>>%3
    lea           offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))]  ; offy*stride+offx

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, unused1, unused2, unused3, luma, lstride

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
%%loop_y:
    ; luma_src
%if %2
    mova            xm2, [lumaq+lstrideq*0+ 0]
    vinserti128      m2, [lumaq+lstrideq*0+32], 1
    mova            xm4, [lumaq+lstrideq*0+16]
    vinserti128      m4, [lumaq+lstrideq*0+48], 1
    mova            xm3, [lumaq+lstrideq*(1<<%3)+ 0]
    vinserti128      m3, [lumaq+lstrideq*(1<<%3)+32], 1
    mova            xm5, [lumaq+lstrideq*(1<<%3)+16]
    vinserti128      m5, [lumaq+lstrideq*(1<<%3)+48], 1
    phaddw           m2, m4
    phaddw           m3, m5
    pxor             m4, m4
    pavgw            m2, m4
    pavgw            m3, m4
%elif %1
    mova             m2, [lumaq+ 0]
    mova             m3, [lumaq+32]
%endif
%if %1
    mova             m0, [srcq]
%if %2
    mova             m1, [srcq+strideq]
%else
    mova             m1, [srcq+32]
%endif
    punpckhwd        m4, m2, m0
    punpcklwd        m2, m0
    punpckhwd        m5, m3, m1
    punpcklwd        m3, m1                 ; { luma, chroma }
    REPX {pmaddwd x, m14}, m4, m2, m5, m3
    REPX {paddd   x, m15}, m4, m2, m5, m3
    REPX {psrad   x, 6  }, m4, m2, m5, m3
    packusdw         m2, m4
    packusdw         m3, m5
    pminuw           m2, m10
    pminuw           m3, m10                ; clip_pixel()
%elif %2
    pand             m2, m10
    pand             m3, m10
%else
    pand             m2, m10, [lumaq+ 0]
    pand             m3, m10, [lumaq+32]
%endif

    ; scaling[luma_src]
    vpbroadcastd     m7, [pd_m65536]
    pandn            m4, m7, m2
    mova             m6, m7
    vpgatherdd       m5, [scalingq+m4-0], m7
    psrld            m2, 16
    mova             m7, m6
    vpgatherdd       m4, [scalingq+m2-2], m6
    pblendw          m4, m5, 0x55
    pandn            m5, m7, m3
    mova             m6, m7
    vpgatherdd       m2, [scalingq+m5-0], m7
    psrld            m3, 16
    vpgatherdd       m5, [scalingq+m3-2], m6
    pblendw          m5, m2, 0x55

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m4, [grain_lutq+offxyq*2]
%if %2
    pmulhrsw         m5, [grain_lutq+offxyq*2+82*2]
%else
    pmulhrsw         m5, [grain_lutq+offxyq*2+32]
%endif

    ; dst = clip_pixel(src, noise)
%if %1
    paddw            m0, m4
    paddw            m1, m5
%else
    paddw            m0, m4, [srcq]
%if %2
    paddw            m1, m5, [srcq+strideq]
%else
    paddw            m1, m5, [srcq+32]
%endif
%endif
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova         [dstq], m0
%if %2
    mova [dstq+strideq], m1
    lea            srcq, [srcq+strideq*2]
    lea            dstq, [dstq+strideq*2]
    lea           lumaq, [lumaq+lstrideq*(2<<%3)]
%else
    mova      [dstq+32], m1
    add            srcq, strideq
    add            dstq, strideq
    add           lumaq, lstrideq
%endif
    add      grain_lutq, 82*(2<<%2)
%if %2
    sub              hb, 2
%else
    dec              hb
%endif
    jg %%loop_y
    add              wq, 32>>%2
    jge .end
    mov            srcq, r9mp
    mov            dstq, r11mp
    mov           lumaq, r12mp
    lea            srcq, [srcq+wq*2]
    lea            dstq, [dstq+wq*2]
    lea           lumaq, [lumaq+wq*(2<<%2)]
    cmp byte [fg_dataq+FGData.overlap_flag], 0
    je %%loop_x
    cmp       dword r8m, 0 ; sby
    jne %%loop_x_hv_overlap

    ; horizontal overlap (without vertical overlap)
%%loop_x_h_overlap:
    rorx             r6, seeq, 1
    or             seed, 0xEFF4
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d               ; updated seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, unused1, unused2, luma, lstride

    lea     left_offxyd, [offyq+(32>>%2)]         ; previous column's offy*stride+offx
    rorx          offyd, seed, 8
    rorx          offxq, seeq, 12
    and           offyd, 0xf
    imul          offyd, 164>>%3
    lea           offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)]  ; offy*stride+offx

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, left_offxy, unused1, unused2, luma, lstride

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
%%loop_y_h_overlap:
    ; luma_src
%if %2
    mova            xm2, [lumaq+lstrideq*0+ 0]
    vinserti128      m2, [lumaq+lstrideq*0+32], 1
    mova            xm4, [lumaq+lstrideq*0+16]
    vinserti128      m4, [lumaq+lstrideq*0+48], 1
    mova            xm3, [lumaq+lstrideq*(1<<%3)+ 0]
    vinserti128      m3, [lumaq+lstrideq*(1<<%3)+32], 1
    mova            xm5, [lumaq+lstrideq*(1<<%3)+16]
    vinserti128      m5, [lumaq+lstrideq*(1<<%3)+48], 1
    phaddw           m2, m4
    phaddw           m3, m5
    pxor             m4, m4
    pavgw            m2, m4
    pavgw            m3, m4
%elif %1
    mova             m2, [lumaq]
    mova             m3, [lumaq+32]
%endif
%if %1
    mova             m0, [srcq]
%if %2
    mova             m1, [srcq+strideq]
%else
    mova             m1, [srcq+32]
%endif
    punpckhwd        m4, m2, m0
    punpcklwd        m2, m0
    punpckhwd        m5, m3, m1
    punpcklwd        m3, m1                 ; { luma, chroma }
    REPX {pmaddwd x, m14}, m4, m2, m5, m3
    REPX {paddd   x, m15}, m4, m2, m5, m3
    REPX {psrad   x, 6  }, m4, m2, m5, m3
    packusdw         m2, m4
    packusdw         m3, m5
    pminuw           m2, m10                ; clip_pixel()
    pminuw           m3, m10
%elif %2
    pand             m2, m10
    pand             m3, m10
%else
    pand             m2, m10, [lumaq+ 0]
    pand             m3, m10, [lumaq+32]
%endif

    ; scaling[luma_src]
    vpbroadcastd     m7, [pd_m65536]
    pandn            m4, m7, m2
    mova             m6, m7
    vpgatherdd       m5, [scalingq+m4-0], m7
    psrld            m2, 16
    mova             m7, m6
    vpgatherdd       m4, [scalingq+m2-2], m6
    pblendw          m4, m5, 0x55
    pandn            m5, m7, m3
    mova             m6, m7
    vpgatherdd       m2, [scalingq+m5-0], m7
    psrld            m3, 16
    vpgatherdd       m5, [scalingq+m3-2], m6
    pblendw          m5, m2, 0x55

    ; grain = grain_lut[offy+y][offx+x]
    movu             m2, [grain_lutq+offxyq*2]
%if %2
    movu             m3, [grain_lutq+offxyq*2+82*2]
%else
    movu             m3, [grain_lutq+offxyq*2+32]
%endif
    movd            xm6, [grain_lutq+left_offxyq*2]
%if %2
    pinsrw          xm6, [grain_lutq+left_offxyq*2+82*2], 2 ; {left0, left1}
    punpckldq       xm7, xm2, xm3           ; {cur0, cur1}
    punpcklwd       xm6, xm7                ; {left0, cur0, left1, cur1}
%else
    punpcklwd       xm6, xm2
%endif
%if %1
%if %2
    vpbroadcastq    xm7, [pw_23_22]
%else
    movq            xm7, [pw_27_17_17_27]
%endif
    pmaddwd         xm6, xm7
    vpbroadcastd    xm7, [pd_16]
    paddd           xm6, xm7
%else
    pmaddwd         xm6, xm15
    paddd           xm6, xm14
%endif
    psrad           xm6, 5
    packssdw        xm6, xm6
    pmaxsw          xm6, xm8
    pminsw          xm6, xm9
    vpblendd         m2, m6, 0x01
%if %2
    pshuflw         xm6, xm6, q1032
    vpblendd         m3, m6, 0x01
%endif

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m2, m4
    pmulhrsw         m3, m5

    ; dst = clip_pixel(src, noise)
%if %1
    paddw            m0, m2
    paddw            m1, m3
%else
    paddw            m0, m2, [srcq]
%if %2
    paddw            m1, m3, [srcq+strideq]
%else
    paddw            m1, m3, [srcq+32]
%endif
%endif
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova         [dstq], m0
%if %2
    mova [dstq+strideq], m1
    lea            srcq, [srcq+strideq*2]
    lea            dstq, [dstq+strideq*2]
    lea           lumaq, [lumaq+lstrideq*(2<<%3)]
%else
    mova      [dstq+32], m1
    add            srcq, strideq
    add            dstq, strideq
    add           lumaq, r10mp
%endif
    add      grain_lutq, 82*(2<<%2)
%if %2
    sub              hb, 2
%else
    dec              hb
%endif
    jg %%loop_y_h_overlap
    add              wq, 32>>%2
    jge .end
    mov            srcq, r9mp
    mov            dstq, r11mp
    mov           lumaq, r12mp
    lea            srcq, [srcq+wq*2]
    lea            dstq, [dstq+wq*2]
    lea           lumaq, [lumaq+wq*(2<<%2)]
    cmp       dword r8m, 0 ; sby
    jne %%loop_x_hv_overlap
    jmp %%loop_x_h_overlap

%%vertical_overlap:
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \
                sby, see, unused1, unused2, unused3, lstride

    movzx          sbyd, sbyb
    imul           seed, [fg_dataq+FGData.seed], 0x00010001
    imul            r7d, sbyd, 173 * 0x00010001
    imul           sbyd, 37 * 0x01000100
    add             r7d, (105 << 16) | 188
    add            sbyd, (178 << 24) | (141 << 8)
    and             r7d, 0x00ff00ff
    and            sbyd, 0xff00ff00
    xor            seed, r7d
    xor            seed, sbyd               ; (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, unused1, top_offxy, unused2, luma, lstride

    mov           lumaq, r9mp
    mov        lstrideq, r10mp
    lea             r10, [srcq+wq*2]
    lea             r11, [dstq+wq*2]
    lea             r12, [lumaq+wq*(2<<%2)]
    mov            r9mp, r10
    mov           r11mp, r11
    mov           r12mp, r12
    neg              wq

%%loop_x_v_overlap:
    ; we assume from the block above that bits 8-15 of r7d are zero'ed
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp            r7b                     ; parity of top_seed
    shr            seed, 16
    shl             r7d, 16
    test           seeb, seeh
    setp            r7b                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor             r7d, r6d
    rorx           seed, r7d, 1             ; updated (cur_seed << 16) | top_seed

    rorx          offyd, seed, 8
    rorx          offxd, seed, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164>>%3
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, unused1, top_offxy, unused2, luma, lstride

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
    movzx    top_offxyd, offxyw
    shr          offxyd, 16
%if %2 == 0
    lea             r10, [pw_27_17_17_27]
%endif
%%loop_y_v_overlap:
    ; luma_src
%if %2
    mova            xm2, [lumaq+lstrideq*0+ 0]
    vinserti128      m2, [lumaq+lstrideq*0+32], 1
    mova            xm4, [lumaq+lstrideq*0+16]
    vinserti128      m4, [lumaq+lstrideq*0+48], 1
    mova            xm3, [lumaq+lstrideq*(1<<%3)+ 0]
    vinserti128      m3, [lumaq+lstrideq*(1<<%3)+32], 1
    mova            xm5, [lumaq+lstrideq*(1<<%3)+16]
    vinserti128      m5, [lumaq+lstrideq*(1<<%3)+48], 1
    phaddw           m2, m4
    phaddw           m3, m5
    pxor             m4, m4
    pavgw            m2, m4
    pavgw            m3, m4
%elif %1
    mova             m2, [lumaq]
    mova             m3, [lumaq+32]
%endif
%if %1
    mova             m0, [srcq]
%if %2
    mova             m1, [srcq+strideq]
%else
    mova             m1, [srcq+32]
%endif
    punpckhwd        m4, m2, m0
    punpcklwd        m2, m0
    punpckhwd        m5, m3, m1
    punpcklwd        m3, m1                 ; { luma, chroma }
    REPX {pmaddwd x, m14}, m4, m2, m5, m3
    REPX {paddd   x, m15}, m4, m2, m5, m3
    REPX {psrad   x, 6  }, m4, m2, m5, m3
    packusdw         m2, m4
    packusdw         m3, m5
    pminuw           m2, m10                ; clip_pixel()
    pminuw           m3, m10
%elif %2
    pand             m2, m10
    pand             m3, m10
%else
    pand             m2, m10, [lumaq+ 0]
    pand             m3, m10, [lumaq+32]
%endif

    ; scaling[luma_src]
    vpbroadcastd     m7, [pd_m65536]
    pandn            m4, m7, m2
    mova             m6, m7
    vpgatherdd       m5, [scalingq+m4-0], m7
    psrld            m2, 16
    mova             m7, m6
    vpgatherdd       m4, [scalingq+m2-2], m6
    pblendw          m4, m5, 0x55
    pandn            m5, m7, m3
    mova             m6, m7
    vpgatherdd       m2, [scalingq+m5-0], m7
    psrld            m3, 16
    vpgatherdd       m5, [scalingq+m3-2], m6
    pblendw          m5, m2, 0x55

    ; grain = grain_lut[offy+y][offx+x]
    movu             m6, [grain_lutq+offxyq*2]
    movu             m3, [grain_lutq+top_offxyq*2]
    punpcklwd        m2, m3, m6
    punpckhwd        m3, m6                 ; { top, cur }
%if %3
    vpbroadcastd     m0, [pw_23_22]
%elif %2
    vpbroadcastd     m0, [pw_27_17_17_27]
%else
    vpbroadcastd     m0, [r10]
%endif
    REPX {pmaddwd x, m0}, m2, m3
%if %1
    vpbroadcastd     m1, [pd_16]
    REPX  {paddd x, m1}, m2, m3
%else
    REPX {paddd x, m14}, m2, m3
%endif
    REPX   {psrad x, 5}, m2, m3
    packssdw         m2, m3
%if %2
    movu             m3, [grain_lutq+offxyq*2+82*2]
%else
    movu             m3, [grain_lutq+offxyq*2+32]
%endif
%if %3
    pmaxsw           m2, m8
    pminsw           m2, m9
%else
%if %2
    movu             m7, [grain_lutq+top_offxyq*2+82*2]
    punpckhwd        m6, m3, m7             ; { cur, top }
    punpcklwd        m3, m7
%else
    movu             m7, [grain_lutq+top_offxyq*2+32]
    punpckhwd        m6, m7, m3
    punpcklwd        m3, m7, m3             ; { top, cur }
%endif
    pmaddwd          m6, m0
    pmaddwd          m3, m0
%if %1
    paddd            m6, m1
    paddd            m3, m1
%else
    paddd            m6, m14
    paddd            m3, m14
%endif
    psrad            m6, 5
    psrad            m3, 5
    packssdw         m3, m6
    pmaxsw           m2, m8
    pmaxsw           m3, m8
    pminsw           m2, m9
    pminsw           m3, m9
%endif

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m2, m4
    pmulhrsw         m3, m5

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2, [srcq]
%if %2
    paddw            m1, m3, [srcq+strideq]
%else
    paddw            m1, m3, [srcq+32]
%endif
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova         [dstq], m0
%if %2
    mova [dstq+strideq], m1
    sub              hb, 2
%else
    mova      [dstq+32], m1
    dec              hb
%endif
    jle %%end_y_v_overlap
%if %2
    lea            srcq, [srcq+strideq*2]
    lea            dstq, [dstq+strideq*2]
    lea           lumaq, [lumaq+lstrideq*(2<<%3)]
%else
    add            srcq, strideq
    add            dstq, strideq
    add           lumaq, lstrideq
%endif
    add      grain_lutq, 82*(2<<%2)
%if %2
    jmp %%loop_y
%else
    add              hd, 0x80000000
    jc %%loop_y
    add             r10, 4
    jmp %%loop_y_v_overlap
%endif
%%end_y_v_overlap:
    add              wq, 32>>%2
    jge .end
    mov            srcq, r9mp
    mov            dstq, r11mp
    mov           lumaq, r12mp
    lea            srcq, [srcq+wq*2]
    lea            dstq, [dstq+wq*2]
    lea           lumaq, [lumaq+wq*(2<<%2)]

    ; since fg_dataq.overlap is guaranteed to be set, we never jump
    ; back to .loop_x_v_overlap, and instead always fall-through to
    ; h+v overlap
%%loop_x_hv_overlap:
    ; we assume from the block above that bits 8-15 of r7d are zero'ed
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp            r7b                     ; parity of top_seed
    shr            seed, 16
    shl             r7d, 16
    test           seeb, seeh
    setp            r7b                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor             r7d, r6d
    rorx           seed, r7d, 1             ; updated (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride

%if %2 == 0
    lea             r14, [pw_27_17_17_27]
%endif
    lea  topleft_offxyq, [top_offxyq+(32>>%2)]
    lea     left_offxyq, [offyq+(32>>%2)]
    rorx          offyd, seed, 8
    rorx          offxd, seed, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164>>%3
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]

    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
                h, offxy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride

    mov      grain_lutq, grain_lutmp
    mov              hd, hm
    movzx    top_offxyd, offxyw
    shr          offxyd, 16
%%loop_y_hv_overlap:
    ; luma_src
%if %2
    mova            xm2, [lumaq+lstrideq*0+ 0]
    vinserti128      m2, [lumaq+lstrideq*0+32], 1
    mova            xm4, [lumaq+lstrideq*0+16]
    vinserti128      m4, [lumaq+lstrideq*0+48], 1
    mova            xm3, [lumaq+lstrideq*(1<<%3)+ 0]
    vinserti128      m3, [lumaq+lstrideq*(1<<%3)+32], 1
    mova            xm5, [lumaq+lstrideq*(1<<%3)+16]
    vinserti128      m5, [lumaq+lstrideq*(1<<%3)+48], 1
    phaddw           m2, m4
    phaddw           m3, m5
    pxor             m4, m4
    pavgw            m2, m4
    pavgw            m3, m4
%elif %1
    mova             m2, [lumaq]
    mova             m3, [lumaq+32]
%endif
%if %1
    mova             m0, [srcq]
%if %2
    mova             m1, [srcq+strideq]
%else
    mova             m1, [srcq+32]
%endif
    punpckhwd        m4, m2, m0
    punpcklwd        m2, m0
    punpckhwd        m5, m3, m1
    punpcklwd        m3, m1                 ; { luma, chroma }
    REPX {pmaddwd x, m14}, m4, m2, m5, m3
    REPX {paddd   x, m15}, m4, m2, m5, m3
    REPX {psrad   x, 6  }, m4, m2, m5, m3
    packusdw         m2, m4
    packusdw         m3, m5
    pminuw           m2, m10                ; clip_pixel()
    pminuw           m3, m10
%elif %2
    pand             m2, m10
    pand             m3, m10
%else
    pand             m2, m10, [lumaq+ 0]
    pand             m3, m10, [lumaq+32]
%endif

    ; scaling[luma_src]
    vpbroadcastd     m7, [pd_m65536]
    pandn            m4, m7, m2
    mova             m6, m7
    vpgatherdd       m5, [scalingq+m4-0], m7
    psrld            m2, 16
    mova             m7, m6
    vpgatherdd       m4, [scalingq+m2-2], m6
    pblendw          m4, m5, 0x55
    pandn            m5, m7, m3
    mova             m6, m7
    vpgatherdd       m2, [scalingq+m5-0], m7
    psrld            m3, 16
    vpgatherdd       m5, [scalingq+m3-2], m6
    pblendw          m5, m2, 0x55

    ; grain = grain_lut[offy+y][offx+x]
    movu             m0, [grain_lutq+offxyq*2]
    movd            xm2, [grain_lutq+left_offxyq*2]
    movu             m6, [grain_lutq+top_offxyq*2]
%if %2
    pinsrw          xm2, [grain_lutq+left_offxyq*2+82*2], 2
    movu             m3, [grain_lutq+offxyq*2+82*2]
    punpckldq       xm1, xm0, xm3           ; { cur0, cur1 }
%if %3
    vinserti128      m2, [grain_lutq+topleft_offxyq*2], 1 ; { left0, left1, top/left }
    vinserti128      m1, [grain_lutq+top_offxyq*2], 1     ; { cur0, cur1, top0 }
%else
    vinserti128      m2, [grain_lutq+topleft_offxyq*2+82*2], 1
    vpbroadcastd     m7, [grain_lutq+topleft_offxyq*2]
    vpblendd         m2, m7, 0x20
    movd            xm7, [grain_lutq+top_offxyq*2+82*2]
    punpckldq       xm7, xm6
    vinserti128      m1, xm7, 1
    movu             m7, [grain_lutq+top_offxyq*2+82*2]
%endif
    punpcklwd        m2, m1                 ; { cur, left }
%if %1
    vpbroadcastq     m1, [pw_23_22]
    pmaddwd          m2, m1
    vpbroadcastd     m1, [pd_16]
    paddd            m2, m1
    psrad            m2, 5
    packssdw         m2, m2
    vpermq           m2, m2, q3120
%else
    pmaddwd          m2, m15
    paddd            m2, m14
    psrad            m2, 5
    vextracti128    xm1, m2, 1
    packssdw        xm2, xm1
%endif
%else
    pinsrd          xm2, [grain_lutq+topleft_offxyq*2], 1
    movu             m3, [grain_lutq+offxyq*2+32]
    movu             m7, [grain_lutq+top_offxyq*2+32]
    punpckldq       xm1, xm0, xm6
    punpcklwd       xm2, xm1                ; { cur, left }
%if %1
    movddup         xm1, [pw_27_17_17_27]
    pmaddwd         xm2, xm1
    vpbroadcastd     m1, [pd_16]
    paddd           xm2, xm1
%else
    pmaddwd         xm2, xm15
    paddd           xm2, xm14
%endif
    psrad           xm2, 5
    packssdw        xm2, xm2
%endif
    pmaxsw          xm2, xm8
    pminsw          xm2, xm9
    vpblendd         m0, m2, 0x01
%if %2
    pshufd          xm2, xm2, q0321
    vpblendd         m3, m2, 0x01
%if %3 == 0
    pshufd          xm2, xm2, q0321
    vpblendd         m7, m2, 0x01
%endif
%endif
    pshuflw         xm2, xm2, q1032
    vpblendd         m2, m6, 0xfe
    punpckhwd        m6, m0                 ; { top, cur }
    punpcklwd        m2, m0
%if %3
    vpbroadcastd     m0, [pw_23_22]
%elif %2
    vpbroadcastd     m0, [pw_27_17_17_27]
%else
    vpbroadcastd     m0, [r14]
%endif
    pmaddwd          m6, m0
    pmaddwd          m2, m0
%if %1
    paddd            m6, m1
    paddd            m2, m1
%else
    paddd            m6, m14
    paddd            m2, m14
%endif
    psrad            m6, 5
    psrad            m2, 5
    packssdw         m2, m6

%if %3
    pmaxsw           m2, m8
    pminsw           m2, m9
%else
%if %2
    punpckhwd        m6, m3, m7
    punpcklwd        m3, m7                 ; { cur, top }
%else
    punpckhwd        m6, m7, m3
    punpcklwd        m3, m7, m3             ; { top, cur }
%endif
    REPX {pmaddwd x, m0}, m6, m3
%if %1
    REPX  {paddd x, m1}, m6, m3
%else
    REPX {paddd x, m14}, m6, m3
%endif
    REPX   {psrad x, 5}, m6, m3
    packssdw         m3, m6
    pmaxsw           m2, m8
    pmaxsw           m3, m8
    pminsw           m2, m9
    pminsw           m3, m9
%endif

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmaddubsw        m4, m11
    pmaddubsw        m5, m11
    paddw            m4, m4
    paddw            m5, m5
    pmulhrsw         m2, m4
    pmulhrsw         m3, m5

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2, [srcq]
%if %2
    paddw            m1, m3, [srcq+strideq]
%else
    paddw            m1, m3, [srcq+32]
%endif
    pmaxsw           m0, m12
    pmaxsw           m1, m12
    pminsw           m0, m13
    pminsw           m1, m13
    mova         [dstq], m0
%if %2
    mova [dstq+strideq], m1
    lea            srcq, [srcq+strideq*2]
    lea            dstq, [dstq+strideq*2]
    lea           lumaq, [lumaq+lstrideq*(2<<%3)]
%else
    mova      [dstq+32], m1
    add            srcq, strideq
    add            dstq, strideq
    add           lumaq, r10mp
%endif
    add      grain_lutq, 82*(2<<%2)
%if %2
    sub              hb, 2
    jg %%loop_y_h_overlap
%else
    dec              hb
    jle %%end_y_hv_overlap
    add              hd, 0x80000000
    jc %%loop_y_h_overlap
    add             r14, 4
    jmp %%loop_y_hv_overlap
%endif
%%end_y_hv_overlap:
    add              wq, 32>>%2
    jge .end
    mov            srcq, r9mp
    mov            dstq, r11mp
    mov           lumaq, r12mp
    lea            srcq, [srcq+wq*2]
    lea            dstq, [dstq+wq*2]
    lea           lumaq, [lumaq+wq*(2<<%2)]
    jmp %%loop_x_hv_overlap
%endmacro

    %%FGUV_32x32xN_LOOP 1, %2, %3
.csfl:
    %%FGUV_32x32xN_LOOP 0, %2, %3
.end:
    RET
%endmacro

GEN_GRAIN_UV_FN 420, 1, 1
FGUV_FN 420,         1, 1
GEN_GRAIN_UV_FN 422, 1, 0
FGUV_FN 422,         1, 0
GEN_GRAIN_UV_FN 444, 0, 0
FGUV_FN 444,         0, 0

%endif ; ARCH_X86_64