;******************************************************************************
;* Copyright (c) 2025-2026 Niklas Haas
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************

%include "ops_include.asm"

SECTION_RODATA

align 16
read8_unpack2:  db   0,  2,  4,  6,  8, 10, 12, 14,  1,  3,  5,  7,  9, 11, 13, 15
read8_unpack3:  db   0,  3,  6,  9,  1,  4,  7, 10,  2,  5,  8, 11, -1, -1, -1, -1
read8_unpack4:  db   0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15
read16_unpack2: db   0,  1,  4,  5,  8,  9, 12, 13,  2,  3,  6,  7, 10, 11, 14, 15
read16_unpack3: db   0,  1,  6,  7,  2,  3,  8,  9,  4,  5, 10, 11, -1, -1, -1, -1
read16_unpack4: db   0,  1,  8,  9,  2,  3, 10, 11,  4,  5, 12, 13,  6,  7, 14, 15
write8_pack2:   db   0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15
write8_pack3:   db   0,  4,  8,  1,  5,  9,  2,  6, 10,  3,  7, 11, -1, -1, -1, -1
write16_pack3:  db   0,  1,  4,  5,  8,  9,  2,  3,  6,  7, 10, 11, -1, -1, -1, -1
%define write8_pack4  read8_unpack4
%define write16_pack4 read16_unpack2
%define write16_pack2 read16_unpack4

bits_mask:      db 128, 64, 32, 16,  8,  4,  2,  1,128, 64, 32, 16,  8,  4,  2,  1
bits_reverse:   db   7,  6,  5,  4,  3,  2,  1,  0, 15, 14, 13, 12, 11, 10,  9,  8
nibbles_pack:   times 8 dw 0x0110

swap16:     db  1,  0,  3,  2,  5,  4,  7,  6,  9,  8, 11, 10, 13, 12, 15, 14
swap32:     db  3,  2,  1,  0,  7,  6,  5,  4, 11, 10,  9,  8, 15, 14, 13, 12
expand16:   db  0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14
expand32:   db  0, 0, 0, 0, 4, 4, 4, 4, 8, 8,  8,  8, 12, 12, 12, 12

align 32
bits_shuf: times 8 db 0
           times 8 db 1
           times 8 db 2
           times 8 db 3

mask1: times 32 db 0x01
mask2: times 32 db 0x03
mask3: times 32 db 0x07
mask4: times 32 db 0x0F

const1b  equ mask1
const1w: times 16 dw 0x1

SECTION .text

;---------------------------------------------------------
; Helper macros for BITS-dependent sized instructions

%macro bitfn 2+
    %if BITS == 8
        %1%+b %2
    %elif BITS == 16
        %1%+w %2
    %elif BITS == 32
        %1%+d %2
    %endif
%endmacro

%define psllx           bitfn psll,
%define psrlx           bitfn psrl,
%define pcmpeqx         bitfn pcmpeq,
%define pmullx          bitfn pmull,
%define vpbroadcastx    bitfn vpbroadcast,

;---------------------------------------------------------
; Planar reads / writes

%macro READ_PLANAR 0
IF X,   movu mx, [in0q]
IF Y,   movu my, [in1q]
IF Z,   movu mz, [in2q]
IF W,   movu mw, [in3q]
%if V2
IF X,   movu mx2, [in0q + mmsize]
IF Y,   movu my2, [in1q + mmsize]
IF Z,   movu mz2, [in2q + mmsize]
IF W,   movu mw2, [in3q + mmsize]
%endif
        LOAD_CONT tmp0q
IF X,   add in0q, BLOCK_SIZE
IF Y,   add in1q, BLOCK_SIZE
IF Z,   add in2q, BLOCK_SIZE
IF W,   add in3q, BLOCK_SIZE
        CONTINUE tmp0q
%endmacro

%macro READ_NIBBLE 0
assert COMPS == 1
        VBROADCASTI128 m12, [mask4]
        LOAD_CONT tmp0q
        pmovzxbw mx,  [in0q]
IF V2,  pmovzxbw mx2, [in0q + (mmsize >> 1)]
        add in0q, BLOCK_WIDTH >> 1
        psllw m8, mx, 8     ; { 00 AB 00 CD ... }
        psrlw mx, 4         ; { 0A 00 0C 00 ... }
        pand m8, m12        ; { 00 0B 00 0D ... }
        por mx, m8          ; { 0A 0B 0C 0D ... }
%if V2
        psllw m9, mx2, 8
        psrlw mx2, 4
        pand m9, m12
        por mx2, m9
%endif
        CONTINUE tmp0q
%endmacro

%macro WRITE_NIBBLE 0
assert COMPS == 1
        VBROADCASTI128 m12, [nibbles_pack]
        pmaddubsw mx, m12
        packuswb mx, mx
%if V2
        pmaddubsw mx2, m12
        packuswb mx2, mx2
%endif
%if cpuflag(avx2)
        vpermq mx, mx, q3120
IF V2,  vpermq mx2, mx2, q3120
%endif
        movu [out0q], xmx
IF V2,  movu [out0q + (mmsize >> 1)], xmx2
        add out0q, BLOCK_WIDTH >> 1
        RET
%endmacro

%macro READ_BIT 0
assert COMPS == 1
%if cpuflag(avx2)
        vpbroadcastd mx,  [in0q]
IF V2,  vpbroadcastd mx2, [in0q + 4]
%else
        movd mx, [in0q]
IF V2,  movd mx2, [in0q + 2]
%endif
        mova m8, [bits_shuf]
        VBROADCASTI128 m9,  [bits_mask]
        VBROADCASTI128 m10, [const1b]
        LOAD_CONT tmp0q
        add in0q, BLOCK_WIDTH >> 3
        pshufb mx,  m8
IF V2,  pshufb mx2, m8
        pand mx,  m9
IF V2,  pand mx2, m9
        pcmpeqb mx,  m9
IF V2,  pcmpeqb mx2, m9
        pand mx,  m10
IF V2,  pand mx2, m10
        CONTINUE tmp0q
%endmacro

%macro WRITE_BIT 0
assert COMPS == 1
        VBROADCASTI128 m8, [bits_reverse]
        psllw mx,  7
IF V2,  psllw mx2, 7
        pshufb mx,  m8
IF V2,  pshufb mx2, m8
        pmovmskb tmp0d, mx
IF V2,  pmovmskb tmp1d, mx2
%if mmsize >= 32
        mov [out0q], tmp0d
IF V2,  mov [out0q + (mmsize >> 3)], tmp1d
%else
        mov [out0q], tmp0w
IF V2,  mov [out0q + (mmsize >> 3)], tmp1w
%endif
        add out0q, BLOCK_WIDTH >> 3
        RET
%endmacro

%macro WRITE_PLANAR 0
IF X,   movu [out0q], mx
IF Y,   movu [out1q], my
IF Z,   movu [out2q], mz
IF W,   movu [out3q], mw
%if V2
IF X,   movu [out0q + mmsize], mx2
IF Y,   movu [out1q + mmsize], my2
IF Z,   movu [out2q + mmsize], mz2
IF W,   movu [out3q + mmsize], mw2
%endif
IF X,   add out0q, BLOCK_WIDTH
IF Y,   add out1q, BLOCK_WIDTH
IF Z,   add out2q, BLOCK_WIDTH
IF W,   add out3q, BLOCK_WIDTH
        RET
%endmacro

;---------------------------------------------------------
; Packed reads, packing and unpacking

%macro read_packed2 0
        movu m8,  [in0q + 0 * mmsize]
        movu m9,  [in0q + 1 * mmsize]
IF V2,  movu m10, [in0q + 2 * mmsize]
IF V2,  movu m11, [in0q + 3 * mmsize]
    %if BITS == 32
        shufps m8, m8, q3120
        shufps m9, m9, q3120
IF V2,  shufps m10, m10, q3120
IF V2,  shufps m11, m11, q3120
    %else
        pshufb m8, m12              ; { X0 Y0 | X1 Y1 }
        pshufb m9, m12              ; { X2 Y2 | X3 Y3 }
IF V2,  pshufb m10, m12
IF V2,  pshufb m11, m12
    %endif
        unpcklpd mx, m8, m9         ; { X0 X2 | X1 X3 }
        unpckhpd my, m8, m9         ; { Y0 Y2 | Y1 Y3 }
IF V2,  unpcklpd mx2, m10, m11
IF V2,  unpckhpd my2, m10, m11
    %if cpuflag(avx2)
        vpermq mx, mx, q3120       ; { X0 X1 | X2 X3 }
        vpermq my, my, q3120       ; { Y0 Y1 | Y2 Y3 }
IF V2,  vpermq mx2, mx2, q3120
IF V2,  vpermq my2, my2, q3120
    %endif
%endmacro

%macro read_packed34 5 ; x, y, z, w, addr
        movu xm8,  [%5 + 0  * COMPS]
        movu xm9,  [%5 + 4  * COMPS]
        movu xm10, [%5 + 8  * COMPS]
        movu xm11, [%5 + 12 * COMPS]
    %if cpuflag(avx2)
        vinserti128 m8,  m8,  [%5 + 16 * COMPS], 1
        vinserti128 m9,  m9,  [%5 + 20 * COMPS], 1
        vinserti128 m10, m10, [%5 + 24 * COMPS], 1
        vinserti128 m11, m11, [%5 + 28 * COMPS], 1
    %endif
    %if BITS == 32
        mova %1, m8
        mova %2, m9
        mova %3, m10
        mova %4, m11
    %else
        pshufb %1, m8,  m12         ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
        pshufb %2, m9,  m12         ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
        pshufb %3, m10, m12         ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
        pshufb %4, m11, m12         ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
    %endif
        punpckldq m8,  %1, %2       ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 }
        punpckldq m9,  %3, %4       ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 }
        punpckhdq m10, %1, %2       ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 }
        punpckhdq m11, %3, %4       ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 }
        punpcklqdq %1, m8, m9       ; { X0 X1 X2 X3 | X4 X5 X6 X7 }
        punpckhqdq %2, m8, m9       ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 }
        punpcklqdq %3, m10, m11     ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 }
IF W,   punpckhqdq %4, m10, m11     ; { W0 W1 W2 W3 | W4 W5 W6 W7 }
%endmacro

%macro READ_PACKED 0
    %if BITS < 32
        VBROADCASTI128 m12, [ read %+ BITS %+ _unpack %+ COMPS ]
    %endif
        LOAD_CONT tmp0q
    %if COMPS == 2
        read_packed2
    %else
        read_packed34 mx,  my,  mz,  mw,  in0q
IF1 V2, read_packed34 mx2, my2, mz2, mw2, in0q + mmsize * COMPS
    %endif
        add in0q, BLOCK_SIZE * COMPS
        CONTINUE tmp0q
%endmacro

%macro read_pal8 6 ; x, y, z, w, palette, index
%if cpuflag(avx2)
        pmovzxbd %1, [%6 + 0]
        pmovzxbd %2, [%6 + 8]
        pmovzxbd %3, [%6 + 16]
        pmovzxbd %4, [%6 + 24]
        vperm2i128 m8,  %1, %3, q0200
        vperm2i128 m9,  %1, %3, q0301
        vperm2i128 m10, %2, %4, q0200
        vperm2i128 m11, %2, %4, q0301
        pcmpeqb m14, m14
        pcmpeqb m15, m15
        vpgatherdd %1, [%5 + 4 * m8], m14
        vpgatherdd %2, [%5 + 4 * m9], m15
        pcmpeqb m14, m14
        pcmpeqb m15, m15
        vpgatherdd %3, [%5 + 4 * m10], m14
        vpgatherdd %4, [%5 + 4 * m11], m15
        pshufb %1, m12
        pshufb %2, m12
        pshufb %3, m12
        pshufb %4, m12
        punpckldq m8,  %1, %2
        punpckldq m9,  %3, %4
        punpckhdq m10, %1, %2
        punpckhdq m11, %3, %4
        punpcklqdq %1, m8, m9
        punpckhqdq %2, m8, m9
        punpcklqdq %3, m10, m11
        punpckhqdq %4, m10, m11
%else ; !cpuflag(avx2)
    %assign i 0
    %rep 16
        movzx tmp1d, byte [%6 + i]
        pinsrb %1, [%5 + 4 * tmp1q + 0], i
        pinsrb %2, [%5 + 4 * tmp1q + 1], i
        pinsrb %3, [%5 + 4 * tmp1q + 2], i
        pinsrb %4, [%5 + 4 * tmp1q + 3], i
    %assign i i+1
    %endrep
%endif
%endmacro

%macro READ_PALETTE 0
assert COMPS == 4
assert BITS  == 8
    %if cpuflag(avx2)
        VBROADCASTI128 m12, [read8_unpack4]
    %endif
        LOAD_CONT tmp0q
        read_pal8 mx,  my,  mz,  mw,  in1q, in0q
IF1 V2, read_pal8 mx2, my2, mz2, mw2, in1q, in0q + mmsize
        add in0q, BLOCK_SIZE
        CONTINUE tmp0q
%endmacro

%macro write_packed2 0
    %if cpuflag(avx2)
        vpermq mx, mx, q3120       ; { X0 X2 | X1 X3 }
        vpermq my, my, q3120       ; { Y0 Y2 | Y1 Y3 }
IF V2,  vpermq mx2, mx2, q3120
IF V2,  vpermq my2, my2, q3120
    %endif
        unpcklpd m8, mx, my        ; { X0 Y0 | X1 Y1 }
        unpckhpd m9, mx, my        ; { X2 Y2 | X3 Y3 }
IF V2,  unpcklpd m10, mx2, my2
IF V2,  unpckhpd m11, mx2, my2
    %if BITS == 32
        shufps m8, m8, q3120
        shufps m9, m9, q3120
IF V2,  shufps m10, m10, q3120
IF V2,  shufps m11, m11, q3120
    %else
        pshufb m8, m12
        pshufb m9, m12
IF V2,  pshufb m10, m12
IF V2,  pshufb m11, m12
    %endif
        movu [out0q + 0 * mmsize], m8
        movu [out0q + 1 * mmsize], m9
IF V2,  movu [out0q + 2 * mmsize], m10
IF V2,  movu [out0q + 3 * mmsize], m11
%endmacro

%macro write_packed34 5 ; x, y, z, w, addr
        punpckldq m8,  %1, %2       ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 }
        punpckldq m9,  %3, %4       ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 }
        punpckhdq m10, %1, %2       ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 }
        punpckhdq m11, %3, %4       ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 }
        punpcklqdq %1, m8, m9       ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 }
        punpckhqdq %2, m8, m9       ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 }
        punpcklqdq %3, m10, m11     ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 }
        punpckhqdq %4, m10, m11     ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 }
    %if BITS == 32
        mova m8,  %1
        mova m9,  %2
        mova m10, %3
        mova m11, %4
    %else
        pshufb m8,  %1, m12
        pshufb m9,  %2, m12
        pshufb m10, %3, m12
        pshufb m11, %4, m12
    %endif
        movu [%5 +  0 * COMPS], xm8
        movu [%5 +  4 * COMPS], xm9
        movu [%5 +  8 * COMPS], xm10
        movu [%5 + 12 * COMPS], xm11
    %if cpuflag(avx2)
        vextracti128 [%5 + 16 * COMPS], m8,  1
        vextracti128 [%5 + 20 * COMPS], m9,  1
        vextracti128 [%5 + 24 * COMPS], m10, 1
        vextracti128 [%5 + 28 * COMPS], m11, 1
    %endif
%endmacro

%macro WRITE_PACKED 0
    %if BITS < 32
        VBROADCASTI128 m12, [ write %+ BITS %+ _pack %+ COMPS ]
    %endif
        LOAD_CONT tmp0q
    %if COMPS == 2
        write_packed2
    %else
        write_packed34 mx,  my,  mz,  mw,  out0q
IF1 V2, write_packed34 mx2, my2, mz2, mw2, out0q + mmsize * COMPS
    %endif
        add out0q, BLOCK_SIZE * COMPS
        RET
%endmacro

%macro PACK 4 ; x, y, z, w
        ; pslld works for all sizes because the input should not overflow
IF %2,  pslld mx, %4+%3+%2
IF %3,  pslld my, %4+%3
IF %4,  pslld mz, %4
IF Y,   por mx, my
IF Z,   por mx, mz
IF W,   por mx, mw
    %if V2
IF %2,  pslld mx2, %4+%3+%2
IF %3,  pslld my2, %4+%3
IF %4,  pslld mz2, %4
IF Y,   por mx2, my2
IF Z,   por mx2, mz2
IF W,   por mx2, mw2
    %endif
        CONTINUE
%endmacro

%macro UNPACK 4 ; x, y, z, w
        LOAD_CONT tmp0q
%if BITS == 8
        assert MASK == SWS_COMP_ELEMS(3)
        pand mz, mx, [mask%3]
        psrld my, mx, %3
        psrld mx, %3+%2
        pand my, [mask%2]
        pand mx, [mask%1]
    %if V2
        pand mz2, mx2, [mask%3]
        psrld my2, mx2, %3
        psrld mx2, %3+%2
        pand my2, [mask%2]
        pand mx2, [mask%1]
    %endif
%else
        ; clear high bits by shifting left
IF W,   psllx mw, mx, BITS - (%4)
IF Z,   psllx mz, mx, BITS - (%4+%3)
IF Y,   psllx my, mx, BITS - (%4+%3+%2)
        psrlx mx, %2+%3+%4
IF Y,   psrlx my, BITS - %2
IF Z,   psrlx mz, BITS - %3
IF W,   psrlx mw, BITS - %4
    %if V2
IF W,   psllx mw2, mx2, BITS - (%4)
IF Z,   psllx mz2, mx2, BITS - (%4+%3)
IF Y,   psllx my2, mx2, BITS - (%4+%3+%2)
        psrlx mx2, %2+%3+%4
IF Y,   psrlx my2, BITS - %2
IF Z,   psrlx mz2, BITS - %3
IF W,   psrlx mw2, BITS - %4
    %endif
%endif
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Pixel type conversions

%macro cast8to16 4 ; reg, reg2, xreg, xreg2
IF V2,  vextracti128 %4, %1, 1
        pmovzxbw %1, %3
IF V2,  pmovzxbw %2, %4
%endmacro

%macro cast16to8 4 ; reg, reg2, xreg, xreg2
%if V2
        packuswb %1, %2
        vpermq %1, %1, q3120
%else
        vextracti128  %4, %1, 1
        packuswb %3, %4
%endif
%endmacro

%macro cast8to32 4 ; reg, reg2, xreg, xreg2
        psrldq %4, %3, 8
        pmovzxbd %1, %3
        pmovzxbd %2, %4
%endmacro

%macro cast32to8 4 ; reg, reg2, xreg, xreg2
        packusdw %1, %2
        vextracti128 %4, %1, 1
        packuswb %3, %4
        vpshufd %3, %3, q3120
%endmacro

%macro cast16to32 4 ; reg, reg2, xreg, xreg2
        vextracti128 %4, %1, 1
        pmovzxwd %1, %3
        pmovzxwd %2, %4
%endmacro

%macro cast32to16 4 ; reg, reg2, xreg, xreg2
        packusdw %1, %2
        vpermq %1, %1, q3120
%endmacro

%macro CAST_TO 0
%ifidn UOP, SWS_UOP_TO_U8
    %assign BITS_TO 8
%elifidn UOP, SWS_UOP_TO_U16
    %assign BITS_TO 16
%else
    %assign BITS_TO 32
%endif

        LOAD_CONT tmp0q
%ifidn TYPE, SWS_PIXEL_F32
IF X,   cvttps2dq mx, mx
IF Y,   cvttps2dq my, my
IF Z,   cvttps2dq mz, mz
IF W,   cvttps2dq mw, mw
IF X,   cvttps2dq mx2, mx2
IF Y,   cvttps2dq my2, my2
IF Z,   cvttps2dq mz2, mz2
IF W,   cvttps2dq mw2, mw2
%endif
%if BITS != BITS_TO
        ; integer size conversion
IF1 X,  cast %+ BITS %+ to %+ BITS_TO mx, mx2, xmx, xmx2
IF1 Y,  cast %+ BITS %+ to %+ BITS_TO my, my2, xmy, xmy2
IF1 Z,  cast %+ BITS %+ to %+ BITS_TO mz, mz2, xmz, xmz2
IF1 W,  cast %+ BITS %+ to %+ BITS_TO mw, mw2, xmw, xmw2
    %if cpuflag(avx2) && (BITS > BITS_TO && !V2 || BITS >= BITS_TO * 4)
        ; clear upper bits after reducing the register size
        vzeroupper ; TMP
    %endif
%endif
%ifidn UOP, SWS_UOP_TO_F32
IF X,   vcvtdq2ps mx, mx
IF Y,   vcvtdq2ps my, my
IF Z,   vcvtdq2ps mz, mz
IF W,   vcvtdq2ps mw, mw
IF X,   vcvtdq2ps mx2, mx2
IF Y,   vcvtdq2ps my2, my2
IF Z,   vcvtdq2ps mz2, mz2
IF W,   vcvtdq2ps mw2, mw2
%endif
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Moving, copying and clearing

%macro MOVE 13 ; num, dst0..dst5, src0..src5
%assign NUM_MOVES %1
%define DST %2
%define SRC %8

        LOAD_CONT tmp0q
%rep NUM_MOVES
        %assign dstidx %2 < 0 ? 8 : %2
        %assign srcidx %8 < 0 ? 8 : %8
        mova m %+ dstidx, m %+ srcidx
    %if V2
        %assign dstidx dstidx + 4
        %assign srcidx srcidx + 4
        mova m %+ dstidx, m %+ srcidx
    %endif
%rotate 1
%endrep
        CONTINUE tmp0q
%endmacro

%macro clear 3 ; idx, reg, reg2
%if SWS_COMP_TEST(ZERO_MASK, %1)
        pxor %2, %2
%elif SWS_COMP_TEST(ONE_MASK, %1)
        pcmpeqb %2, %2
%elif cpuflag(avx)
        vpbroadcastd %2, [implq + SwsOpImpl.priv + 4 * %1]
%else
        movd %2, [implq + SwsOpImpl.priv + 4 * %1]
        pshufd %2, %2, 0
%endif
IF V2,  mova %3, %2
%endmacro

%macro CLEAR 2
%assign ONE_MASK  %1
%assign ZERO_MASK %2
IF1 X,  clear 0, mx, mx2
IF1 Y,  clear 1, my, my2
IF1 Z,  clear 2, mz, mz2
IF1 W,  clear 3, mw, mw2
        CONTINUE
%endmacro

;---------------------------------------------------------
; Bit manipulation

%macro SWAP_BYTES 0
        VBROADCASTI128 m8, [swap %+ BITS]
IF X,   pshufb mx, m8
IF Y,   pshufb my, m8
IF Z,   pshufb mz, m8
IF W,   pshufb mw, m8
%if V2
IF X,   pshufb mx2, m8
IF Y,   pshufb my2, m8
IF Z,   pshufb mz2, m8
IF W,   pshufb mw2, m8
%endif
        CONTINUE
%endmacro

%macro EXPAND_BIT 0
%if BITS == 8
        VBROADCASTI128 m8, [const1b]
%elif BITS == 16
        VBROADCASTI128 m8, [const1w]
%else
        assert 0, EXPAND_BIT is not implemented for 32-bit types
%endif
IF X,   pcmpeqx mx, m8
IF Y,   pcmpeqx my, m8
IF Z,   pcmpeqx mz, m8
IF W,   pcmpeqx mw, m8
%if V2
IF X,   pcmpeqx mx2, m8
IF Y,   pcmpeqx my2, m8
IF Z,   pcmpeqx mz2, m8
IF W,   pcmpeqx mw2, m8
%endif
        CONTINUE
%endmacro

%macro EXPAND_BYTE 0
        assert BITS == 8
%ifidn UOP, SWS_UOP_EXPAND_PAIR
        %assign BITS_TO BITS * 2
%elifidn UOP, SWS_UOP_EXPAND_QUAD
        %assign BITS_TO BITS * 4
%endif
        VBROADCASTI128 m8, [expand %+ BITS_TO]
        LOAD_CONT tmp0q
IF1 X,  cast8to %+ BITS_TO mx, mx2, xmx, xmx2
IF1 Y,  cast8to %+ BITS_TO my, my2, xmy, xmy2
IF1 Z,  cast8to %+ BITS_TO mz, mz2, xmz, xmz2
IF1 W,  cast8to %+ BITS_TO mw, mw2, xmw, xmw2
IF X,   pshufb mx, m8
IF Y,   pshufb my, m8
IF Z,   pshufb mz, m8
IF W,   pshufb mw, m8
    %if V2
IF X,   pshufb mx2, m8
IF Y,   pshufb my2, m8
IF Z,   pshufb mz2, m8
IF W,   pshufb mw2, m8
    %endif
        CONTINUE tmp0q
%endmacro

%macro LSHIFT 1 ; shift
        LOAD_CONT tmp0q
IF X,   psllx mx, %1
IF Y,   psllx my, %1
IF Z,   psllx mz, %1
IF W,   psllx mw, %1
%if V2
IF X,   psllx mx2, %1
IF Y,   psllx my2, %1
IF Z,   psllx mz2, %1
IF W,   psllx mw2, %1
%endif
        CONTINUE tmp0q
%endmacro

%macro RSHIFT 1 ; shift
        LOAD_CONT tmp0q
IF X,   psrlx mx, %1
IF Y,   psrlx my, %1
IF Z,   psrlx mz, %1
IF W,   psrlx mw, %1
%if V2
IF X,   psrlx mx2, %1
IF Y,   psrlx my2, %1
IF Z,   psrlx mz2, %1
IF W,   psrlx mw2, %1
%endif
        CONTINUE tmp0q
%endmacro

;---------------------------------------------------------
; Arithmetic operations

%macro pmull4 6 ; suffix, x, y, z, w, k
IF X,   pmull%1 %2, %6
IF Y,   pmull%1 %3, %6
IF Z,   pmull%1 %4, %6
IF W,   pmull%1 %5, %6
%endmacro

%macro scale8 5 ; mx, my, mz, mw, k
IF X,   mova m8,  %1
IF Y,   mova m9,  %2
IF Z,   mova m10, %3
IF W,   mova m11, %4
IF X,   punpcklbw %1,  m15
IF Y,   punpcklbw %2,  m15
IF Z,   punpcklbw %3,  m15
IF W,   punpcklbw %4,  m15
IF X,   punpckhbw m8,  m15
IF Y,   punpckhbw m9,  m15
IF Z,   punpckhbw m10, m15
IF W,   punpckhbw m11, m15
        pmull4 w, %1, %2, %3,  %4,  m12
        pmull4 w, m8, m9, m10, m11, m12
IF X,   packuswb %1, m8
IF Y,   packuswb %2, m9
IF Z,   packuswb %3, m10
IF W,   packuswb %4, m11
%endmacro

%macro SCALE 0
        LOAD_CONT tmp0q
%if BITS == 8
%if cpuflag(avx2)
        vpbroadcastw m12, [implq + SwsOpImpl.priv]
%else
        movd xm12, [implq + SwsOpImpl.priv]
        SPLATW m12, xm12, 0
%endif
        pxor m15, m15
        scale8 mx,  my,  mz,  mw,  m12
IF1 V2, scale8 mx2, my2, mz2, mw2, m12
%else
        vpbroadcastx m12, [implq + SwsOpImpl.priv]
        pmull4 x, mx,  my,  mz,  mw,  m12
IF1 V2, pmull4 x, mx2, my2, mz2, mw2, m12
%endif
        CONTINUE tmp0q
%endmacro

%macro ADD 0
assert 0, SWS_UOP_ADD is not implemented for integer types
%endmacro

%macro MIN 0
assert 0, SWS_UOP_MIN is not implemented for integer types
%endmacro

%macro MAX 0
assert 0, SWS_UOP_MAX is not implemented for integer types
%endmacro

%macro LINEAR_FMA 0-*
assert 0, SWS_UOP_LINEAR_FMA is not implemented for integer types
%endmacro

%macro DITHER 0-*
assert 0, SWS_UOP_DITHER is not implemented for integer types
%endmacro

;---------------------------------------------------------
; Instantiate above macros to generate all uop kernels

%macro decl_ops 1 ; type
    DECL_%1_READ_PACKED     (READ_PACKED)
    DECL_%1_READ_NIBBLE     (READ_NIBBLE)
    DECL_%1_READ_BIT        (READ_BIT)
    DECL_%1_READ_PALETTE    (READ_PALETTE)
    DECL_%1_WRITE_PACKED    (WRITE_PACKED)
    DECL_%1_WRITE_NIBBLE    (WRITE_NIBBLE)
    DECL_%1_WRITE_BIT       (WRITE_BIT)
    DECL_%1_MOVE            (MOVE)
    DECL_%1_SWAP_BYTES      (SWAP_BYTES)
    DECL_%1_EXPAND_BIT      (EXPAND_BIT)
    DECL_%1_SCALE           (SCALE)
    DECL_%1_ADD             (ADD)
    DECL_%1_MIN             (MIN)
    DECL_%1_MAX             (MAX)
    DECL_%1_UNPACK          (UNPACK)
    DECL_%1_PACK            (PACK)
    DECL_%1_LSHIFT          (LSHIFT)
    DECL_%1_RSHIFT          (RSHIFT)
    DECL_%1_LINEAR_FMA      (LINEAR_FMA)
    DECL_%1_DITHER          (DITHER)
%endmacro

%macro decl_type_invariant 0
    DECL_U8_READ_PLANAR     (READ_PLANAR)
    DECL_U8_WRITE_PLANAR    (WRITE_PLANAR)
    DECL_U8_CLEAR           (CLEAR)
%endmacro

%macro decl_cast_u16 0
    DECL_U8_TO_U16          (CAST_TO)
    DECL_U16_TO_U8          (CAST_TO)
    DECL_U8_EXPAND_PAIR     (EXPAND_BYTE)
%endmacro

%macro decl_cast_u32 0
    DECL_U8_TO_U32          (CAST_TO)
    DECL_U32_TO_U8          (CAST_TO)
    DECL_U16_TO_U32         (CAST_TO)
    DECL_U32_TO_U16         (CAST_TO)
    DECL_U8_EXPAND_QUAD     (EXPAND_BYTE)
%endmacro

%macro decl_cast_f32 0
    DECL_U8_TO_F32          (CAST_TO)
    DECL_F32_TO_U8          (CAST_TO)
    DECL_U16_TO_F32         (CAST_TO)
    DECL_F32_TO_U16         (CAST_TO)
    DECL_U32_TO_F32         (CAST_TO)
    DECL_F32_TO_U32         (CAST_TO)
%endmacro

INIT_XMM sse4
decl_v2 0, decl_ops U8
decl_v2 1, decl_ops U8
decl_v2 0, decl_type_invariant
decl_v2 1, decl_type_invariant

INIT_YMM avx2
decl_v2 0, decl_ops U8
decl_v2 1, decl_ops U8
decl_v2 0, decl_ops U16
decl_v2 1, decl_ops U16
decl_v2 0, decl_type_invariant
decl_v2 1, decl_type_invariant
decl_v2 0, decl_cast_u16
decl_v2 1, decl_cast_u16

decl_v2 1, decl_ops U32
decl_v2 1, decl_cast_u32
decl_v2 1, decl_cast_f32
