/*
 * HEVC Intra Prediction NEON optimizations
 *
 * Copyright (c) 2026 Jun Zhao <barryjzhao@tencent.com>
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "libavutil/aarch64/asm.S"

/* HEVC Intra Prediction NEON functions
 *
 * Internal NEON function signatures — the C dispatch wrappers in
 * hevcpred_init_aarch64.c handle log2_size-based dispatch, so these
 * per-size functions do not take log2_size themselves:
 *
 * pred_dc_NxN:       void (uint8_t *src, const uint8_t *top,
 *                          const uint8_t *left, ptrdiff_t stride, int c_idx)
 * pred_planar_NxN:   void (uint8_t *src, const uint8_t *top,
 *                          const uint8_t *left, ptrdiff_t stride)
 * pred_angular_*_NxN: void (uint8_t *src, const uint8_t *top,
 *                           const uint8_t *left, ptrdiff_t stride,
 *                           int c_idx, int mode)
 *
 * Mode 10 and 26 accept log2_size since they share one entry point per mode.
 */

// =============================================================================
// DC Prediction
// =============================================================================

/*
 * DC prediction algorithm:
 * 1. dc = sum(top[0..size-1]) + sum(left[0..size-1]) + size
 * 2. dc >>= (log2_size + 1)
 * 3. Fill block with dc value
 * 4. If c_idx == 0 && size < 32: smooth edges
 *    - POS(0,0) = (left[0] + 2*dc + top[0] + 2) >> 2
 *    - First row: (top[x] + 3*dc + 2) >> 2
 *    - First col: (left[y] + 3*dc + 2) >> 2
*/

// -----------------------------------------------------------------------------
// pred_dc_4x4_8: DC prediction
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// w4: c_idx
// -----------------------------------------------------------------------------
function ff_hevc_pred_dc_4x4_8_neon, export=1
        // Load top[0..3] and left[0..3]
        ldr             s0, [x1]                // top[0..3]
        ldr             s1, [x2]                // left[0..3]

        // Sum using NEON
        uaddlv          h2, v0.8b               // sum top (only 4 valid bytes)
        uaddlv          h3, v1.8b               // sum left (only 4 valid bytes)
        add             v2.4h, v2.4h, v3.4h     // total sum

        // Add rounding and shift by 3 (urshr = unsigned rounding shift right)
        add             x5, x0, x3, lsl #1      // row 2 address (early for str)
        urshr           v2.4h, v2.4h, #3        // (sum + 4) >> 3
        dup             v2.8b, v2.b[0]          // broadcast dc

        // Store 4 rows
        str             s2, [x0]
        str             s2, [x0, x3]
        str             s2, [x5]
        str             s2, [x5, x3]

        // Edge smoothing for luma only
        cbnz            w4, 2f

        // Compute 3*dc in NEON domain (16-bit)
        uxtl            v3.8h, v2.8b            // widen dc to 16-bit
        add             v6.8h, v3.8h, v3.8h     // 2*dc
        add             v3.8h, v6.8h, v3.8h     // 3*dc

        // Widen top and left to 16-bit
        uxtl            v4.8h, v0.8b            // top[0..3] widened
        uxtl            v5.8h, v1.8b            // left[0..3] widened

        // Corner: (top[0] + left[0] + 2*dc + 2) >> 2
        // First row: (top[x] + 3*dc + 2) >> 2
        // First column: (left[y] + 3*dc + 2) >> 2
        add             v7.4h, v4.4h, v5.4h     // corner: top[x] + left[x] (only lane 0 matters)
        add             v4.8h, v4.8h, v3.8h     // first row: top[x] + 3*dc
        add             v5.8h, v5.8h, v3.8h     // first column: left[y] + 3*dc
        add             v7.4h, v7.4h, v6.4h     // corner: + 2*dc
        rshrn           v4.8b, v4.8h, #2        // first row: (x + 2) >> 2
        rshrn           v5.8b, v5.8h, #2        // first column: (x + 2) >> 2
        rshrn           v7.8b, v7.8h, #2        // corner: (x + 2) >> 2

        // Overwrite corner byte in row result
        ins             v4.b[0], v7.b[0]

        // Store smoothed first row
        str             s4, [x0]

        // Store smoothed column for y=1..3
        add             x5, x0, x3
        add             x6, x0, x3, lsl #1
        add             x7, x5, x3, lsl #1
        st1             {v5.b}[1], [x5]
        st1             {v5.b}[2], [x6]
        st1             {v5.b}[3], [x7]

2:      ret
endfunc

// -----------------------------------------------------------------------------
// pred_dc_8x8_8: DC prediction
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// w4: c_idx
// -----------------------------------------------------------------------------
function ff_hevc_pred_dc_8x8_8_neon, export=1
        // Load top[0..7] and left[0..7]
        ldr             d0, [x1]                // top[0..7]
        ldr             d1, [x2]                // left[0..7]

        // Sum all pixels
        uaddlv          h2, v0.8b               // sum top
        uaddlv          h3, v1.8b               // sum left
        add             v2.4h, v2.4h, v3.4h     // total sum

        // Add rounding and shift by 4
        urshr           v2.4h, v2.4h, #4        // (sum + 8) >> 4
        dup             v2.8b, v2.b[0]          // broadcast dc

        // Check if edge smoothing needed (luma only)
        cbnz            w4, 2f

        // === Luma path: fill + edge smoothing combined ===

        // Compute 3*dc in NEON domain (16-bit)
        uxtl            v3.8h, v2.8b            // widen dc to 16-bit
        add             v6.8h, v3.8h, v3.8h     // 2*dc
        add             v3.8h, v6.8h, v3.8h     // 3*dc

        // Widen top and left to 16-bit
        uxtl            v4.8h, v0.8b
        uxtl            v5.8h, v1.8b

        // Corner: (top[0] + left[0] + 2*dc + 2) >> 2
        // Smoothed first row: (top[x] + 3*dc + 2) >> 2
        // Smoothed column: (left[y] + 3*dc + 2) >> 2
        add             v7.4h, v4.4h, v5.4h     // corner: top[x] + left[x] (only lane 0 matters)
        add             v4.8h, v4.8h, v3.8h     // first row: top[x] + 3*dc
        add             v5.8h, v5.8h, v3.8h     // column: left[y] + 3*dc
        add             v7.4h, v7.4h, v6.4h     // corner: + 2*dc
        rshrn           v4.8b, v4.8h, #2        // first row: (x + 2) >> 2
        rshrn           v5.8b, v5.8h, #2        // column: (x + 2) >> 2
        rshrn           v7.8b, v7.8h, #2        // corner: (x + 2) >> 2
        // Overwrite corner byte
        ins             v4.b[0], v7.b[0]

        // Store row 0 (smoothed)
        str             d4, [x0]

        // Store DC fill for rows 1-7 with pre-computed addresses
        add             x15, x0, x3, lsl #1
        str             d2, [x0, x3]
        add             x5, x0, x3, lsl #2
        str             d2, [x15]
        str             d2, [x15, x3]
        str             d2, [x5]
        add             x15, x5, x3, lsl #1
        str             d2, [x5, x3]
        str             d2, [x15]
        str             d2, [x15, x3]

        // Scatter-store column bytes with pre-computed addresses
        add             x5, x0, x3
        add             x6, x0, x3, lsl #1
        add             x7, x5, x3, lsl #1
        add             x8, x0, x3, lsl #2
        st1             {v5.b}[1], [x5]
        st1             {v5.b}[2], [x6]
        st1             {v5.b}[3], [x7]
        st1             {v5.b}[4], [x8]
        add             x5, x8, x3
        add             x6, x8, x3, lsl #1
        add             x7, x5, x3, lsl #1
        st1             {v5.b}[5], [x5]
        st1             {v5.b}[6], [x6]
        st1             {v5.b}[7], [x7]
        ret

2:      // === Chroma path: plain DC fill ===
        str             d2, [x0]
        add             x15, x0, x3, lsl #1
        str             d2, [x0, x3]
        add             x0, x0, x3, lsl #2
        str             d2, [x15]
        str             d2, [x15, x3]
        str             d2, [x0]
        add             x15, x0, x3, lsl #1
        str             d2, [x0, x3]
        str             d2, [x15]
        str             d2, [x15, x3]
        ret
endfunc

// -----------------------------------------------------------------------------
// pred_dc_16x16_8: DC prediction
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// w4: c_idx
// -----------------------------------------------------------------------------
function ff_hevc_pred_dc_16x16_8_neon, export=1
        // Load top[0..15] and left[0..15]
        ldr             q0, [x1]                // top[0..15]
        ldr             q1, [x2]                // left[0..15]

        // Sum all pixels
        uaddlv          h2, v0.16b              // sum top
        uaddlv          h3, v1.16b              // sum left
        add             v2.4h, v2.4h, v3.4h

        // Add rounding and shift by 5
        urshr           v2.4h, v2.4h, #5        // (sum + 16) >> 5
        dup             v2.16b, v2.b[0]         // broadcast dc

        // Check if edge smoothing needed (luma only)
        cbnz            w4, 2f

        // === Luma path: fill + edge smoothing combined ===

        // Compute 3*dc in NEON domain (16-bit)
        uxtl            v3.8h, v2.8b            // widen dc to 16-bit
        add             v6.8h, v3.8h, v3.8h     // 2*dc
        add             v3.8h, v6.8h, v3.8h     // 3*dc

        // Widen top to 16-bit
        uxtl            v4.8h, v0.8b
        uxtl2           v5.8h, v0.16b

        // Corner: (top[0] + left[0] + 2*dc + 2) >> 2
        // Smoothed first row: (top[x] + 3*dc + 2) >> 2
        uxtl            v7.8h, v1.8b            // widen left[0..7] (reuse for corner lane 0)
        add             v16.4h, v4.4h, v7.4h    // corner: top[x] + left[x] (only lane 0 matters)
        add             v4.8h, v4.8h, v3.8h     // first row lo: top[x] + 3*dc
        add             v5.8h, v5.8h, v3.8h     // first row hi: top[x] + 3*dc
        add             v16.4h, v16.4h, v6.4h   // corner: + 2*dc
        rshrn           v4.8b, v4.8h, #2        // first row lo: >> 2
        rshrn2          v4.16b, v5.8h, #2       // first row hi: >> 2 (smoothed first row)
        rshrn           v16.8b, v16.8h, #2      // corner: >> 2
        // Overwrite corner byte
        ins             v4.b[0], v16.b[0]

        // Smoothed column: (left[y] + 3*dc + 2) >> 2
        uxtl            v5.8h, v1.8b
        uxtl2           v6.8h, v1.16b
        add             v5.8h, v5.8h, v3.8h
        add             v6.8h, v6.8h, v3.8h
        rshrn           v5.8b, v5.8h, #2
        rshrn2          v5.16b, v6.8h, #2       // smoothed column values

        // Store row 0 (smoothed)
        str             q4, [x0]

        // Store DC fill for all 15 remaining rows
        add             x15, x0, x3, lsl #1
        str             q2, [x0, x3]            // row 1
        add             x5, x0, x3, lsl #2
        str             q2, [x15]               // row 2
        str             q2, [x15, x3]           // row 3
        str             q2, [x5]                // row 4
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 5
        add             x5, x5, x3, lsl #2
        str             q2, [x15]               // row 6
        str             q2, [x15, x3]           // row 7
        str             q2, [x5]                // row 8
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 9
        add             x5, x5, x3, lsl #2
        str             q2, [x15]               // row 10
        str             q2, [x15, x3]           // row 11
        str             q2, [x5]                // row 12
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 13
        str             q2, [x15]               // row 14
        str             q2, [x15, x3]           // row 15

        // Now scatter-store column bytes over the DC fill
        add             x5, x0, x3
        add             x6, x0, x3, lsl #1
        add             x7, x5, x3, lsl #1
        add             x8, x0, x3, lsl #2
        st1             {v5.b}[1], [x5]
        st1             {v5.b}[2], [x6]
        st1             {v5.b}[3], [x7]
        st1             {v5.b}[4], [x8]
        add             x5, x8, x3
        add             x6, x8, x3, lsl #1
        add             x7, x5, x3, lsl #1
        add             x9, x8, x3, lsl #2
        st1             {v5.b}[5], [x5]
        st1             {v5.b}[6], [x6]
        st1             {v5.b}[7], [x7]
        st1             {v5.b}[8], [x9]
        add             x5, x9, x3
        add             x6, x9, x3, lsl #1
        add             x7, x5, x3, lsl #1
        add             x8, x9, x3, lsl #2
        st1             {v5.b}[9], [x5]
        st1             {v5.b}[10], [x6]
        st1             {v5.b}[11], [x7]
        st1             {v5.b}[12], [x8]
        add             x5, x8, x3
        add             x6, x8, x3, lsl #1
        add             x7, x5, x3, lsl #1
        st1             {v5.b}[13], [x5]
        st1             {v5.b}[14], [x6]
        st1             {v5.b}[15], [x7]
        ret

2:      // === Chroma path: plain DC fill ===
        str             q2, [x0]                // row 0
        add             x15, x0, x3, lsl #1
        str             q2, [x0, x3]            // row 1
        add             x5, x0, x3, lsl #2
        str             q2, [x15]               // row 2
        str             q2, [x15, x3]           // row 3
        str             q2, [x5]                // row 4
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 5
        add             x5, x5, x3, lsl #2
        str             q2, [x15]               // row 6
        str             q2, [x15, x3]           // row 7
        str             q2, [x5]                // row 8
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 9
        add             x5, x5, x3, lsl #2
        str             q2, [x15]               // row 10
        str             q2, [x15, x3]           // row 11
        str             q2, [x5]                // row 12
        add             x15, x5, x3, lsl #1
        str             q2, [x5, x3]            // row 13
        str             q2, [x15]               // row 14
        str             q2, [x15, x3]           // row 15
        ret
endfunc

// -----------------------------------------------------------------------------
// pred_dc_32x32_8: DC prediction (no edge smoothing)
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// w4: c_idx
// -----------------------------------------------------------------------------
function ff_hevc_pred_dc_32x32_8_neon, export=1
        // Load top[0..31] and left[0..31]
        ldp             q0, q1, [x1]            // top[0..31]
        ldp             q2, q3, [x2]            // left[0..31]

        // Sum all pixels
        uaddlv          h0, v0.16b
        uaddlv          h1, v1.16b
        uaddlv          h2, v2.16b
        uaddlv          h3, v3.16b
        add             v0.4h, v0.4h, v1.4h
        add             v2.4h, v2.4h, v3.4h
        add             v0.4h, v0.4h, v2.4h

        // Add rounding and shift by 6 (urshr = unsigned rounding shift right)
        urshr           v0.4h, v0.4h, #6
        dup             v0.16b, v0.b[0]
        mov             v1.16b, v0.16b

        // Store 32 rows
        mov             w6, #32
2:
        subs            w6, w6, #1
        stp             q0, q1, [x0]
        add             x0, x0, x3
        b.ne            2b

        // No edge smoothing for 32x32 (size >= 32)
        ret
endfunc

// =============================================================================
// Planar Prediction
// =============================================================================

/*
 * Planar prediction algorithm:
 * For each pixel (x, y):
 * POS(x,y) = ((size-1-x)*left[y] + (x+1)*top[size] +
 *             (size-1-y)*top[x] + (y+1)*left[size] + size) >> (log2_size+1)
 */
// -----------------------------------------------------------------------------
// pred_planar_4x4_8: Planar prediction
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// -----------------------------------------------------------------------------
function ff_hevc_pred_planar_4x4_8_neon, export=1
        // Load reference samples
        ldr             s0, [x1]                // top[0..3]
        ldr             s1, [x2]                // left[0..3]
        ldrb            w4, [x1, #4]            // top[4]
        ldrb            w5, [x2, #4]            // left[4]

        // Setup weight vectors for x direction
        movrel          x6, planar_weights_4
        ldp             d4, d5, [x6]            // weights_dec, weights_inc

        // Precompute base[x] = inc[x]*top[4] + 3*top[x] + left[4] + 4
        dup             v6.8b, w4               // top[4]
        umull           v2.8h, v5.8b, v6.8b     // inc[x]*top[4]
        uxtl            v3.8h, v0.8b            // widen top[x]
        ushll           v6.8h, v0.8b, #1        // top[x]<<1
        add             v6.8h, v6.8h, v3.8h     // 3*top[x]
        add             v2.8h, v2.8h, v6.8h     // + inc*top[4]
        dup             v5.8h, w5               // left[4] as 16-bit
        add             v2.8h, v2.8h, v5.8h     // + left[4]  (y=0: (y+1)*left[4]=1*left[4])
        // v2 = base for row 0 (rounding folded into rshrn below)

        // Precompute decrement: top[x] - left[4]
        sub             v5.8h, v3.8h, v5.8h     // decrement = top[x] - left[4]

        // Pre-dup all left values
        dup             v16.8b, v1.b[0]         // left[0]
        dup             v17.8b, v1.b[1]         // left[1]
        dup             v18.8b, v1.b[2]         // left[2]
        dup             v19.8b, v1.b[3]         // left[3]

        // Compute bases for all 4 rows
        mov             v3.16b, v2.16b          // base0 = base
        sub             v20.8h, v2.8h, v5.8h    // base1 = base - dec
        sub             v21.8h, v20.8h, v5.8h   // base2 = base1 - dec
        sub             v22.8h, v21.8h, v5.8h   // base3 = base2 - dec

        // Interleaved row 0/1 computation
        umull           v6.8h, v4.8b, v16.8b    // row0: dec*left[0]
        umull           v7.8h, v4.8b, v17.8b    // row1: dec*left[1]
        add             v6.8h, v6.8h, v3.8h     // row0: + base0
        add             v7.8h, v7.8h, v20.8h    // row1: + base1
        rshrn           v6.8b, v6.8h, #3        // row0: >>3
        rshrn           v7.8b, v7.8h, #3        // row1: >>3

        // Interleaved row 2/3 computation
        umull           v23.8h, v4.8b, v18.8b   // row2: dec*left[2]
        st1             {v6.s}[0], [x0], x3     // store row0
        umull           v24.8h, v4.8b, v19.8b   // row3: dec*left[3]
        st1             {v7.s}[0], [x0], x3     // store row1
        add             v23.8h, v23.8h, v21.8h  // row2: + base2
        add             v24.8h, v24.8h, v22.8h  // row3: + base3
        rshrn           v23.8b, v23.8h, #3      // row2
        rshrn           v24.8b, v24.8h, #3      // row3
        st1             {v23.s}[0], [x0], x3    // store row2
        str             s24, [x0]               // store row3

        ret
endfunc

// -----------------------------------------------------------------------------
// pred_planar_8x8_8: Planar prediction
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// -----------------------------------------------------------------------------
function ff_hevc_pred_planar_8x8_8_neon, export=1
        // Load reference samples
        ldr             d0, [x1]                // top[0..7]
        ldr             d1, [x2]                // left[0..7]
        ldrb            w4, [x1, #8]            // top[8]
        ldrb            w5, [x2, #8]            // left[8]

        // Setup weight vectors
        movrel          x6, planar_weights_8
        ldp             d4, d5, [x6]            // weights_dec, weights_inc

        // Precompute base[x] = inc[x]*top[8] + 7*top[x] + left[8] + 8
        dup             v6.8b, w4               // top[8]
        umull           v2.8h, v5.8b, v6.8b     // inc[x]*top[8]
        uxtl            v3.8h, v0.8b            // widen top[x]
        ushll           v6.8h, v0.8b, #3        // top[x]<<3
        sub             v6.8h, v6.8h, v3.8h     // 7*top[x]
        add             v2.8h, v2.8h, v6.8h     // + inc*top[8]
        dup             v5.8h, w5               // left[8] as 16-bit
        add             v2.8h, v2.8h, v5.8h     // + left[8]
        // v2 = base for row 0 (rounding folded into rshrn below)

        // Precompute decrement: top[x] - left[8]
        sub             v5.8h, v3.8h, v5.8h     // decrement

        // Precompute all 8 row bases to break serial dependency chain
        mov             v16.16b, v2.16b         // base0
        sub             v17.8h, v16.8h, v5.8h   // base1
        sub             v18.8h, v17.8h, v5.8h   // base2
        sub             v19.8h, v18.8h, v5.8h   // base3
        sub             v20.8h, v19.8h, v5.8h   // base4
        sub             v21.8h, v20.8h, v5.8h   // base5
        sub             v22.8h, v21.8h, v5.8h   // base6
        sub             v23.8h, v22.8h, v5.8h   // base7

        // Rows 0-1
        dup             v6.8b, v1.b[0]
        dup             v7.8b, v1.b[1]
        umull           v2.8h, v4.8b, v6.8b
        umull           v3.8h, v4.8b, v7.8b
        add             v2.8h, v2.8h, v16.8h
        add             v3.8h, v3.8h, v17.8h
        rshrn           v2.8b, v2.8h, #4
        rshrn           v3.8b, v3.8h, #4
        st1             {v2.d}[0], [x0], x3
        st1             {v3.d}[0], [x0], x3

        // Rows 2-3
        dup             v6.8b, v1.b[2]
        dup             v7.8b, v1.b[3]
        umull           v2.8h, v4.8b, v6.8b
        umull           v3.8h, v4.8b, v7.8b
        add             v2.8h, v2.8h, v18.8h
        add             v3.8h, v3.8h, v19.8h
        rshrn           v2.8b, v2.8h, #4
        rshrn           v3.8b, v3.8h, #4
        st1             {v2.d}[0], [x0], x3
        st1             {v3.d}[0], [x0], x3

        // Rows 4-5
        dup             v6.8b, v1.b[4]
        dup             v7.8b, v1.b[5]
        umull           v2.8h, v4.8b, v6.8b
        umull           v3.8h, v4.8b, v7.8b
        add             v2.8h, v2.8h, v20.8h
        add             v3.8h, v3.8h, v21.8h
        rshrn           v2.8b, v2.8h, #4
        rshrn           v3.8b, v3.8h, #4
        st1             {v2.d}[0], [x0], x3
        st1             {v3.d}[0], [x0], x3

        // Rows 6-7
        dup             v6.8b, v1.b[6]
        dup             v7.8b, v1.b[7]
        umull           v2.8h, v4.8b, v6.8b
        umull           v3.8h, v4.8b, v7.8b
        add             v2.8h, v2.8h, v22.8h
        add             v3.8h, v3.8h, v23.8h
        rshrn           v2.8b, v2.8h, #4
        rshrn           v3.8b, v3.8h, #4
        st1             {v2.d}[0], [x0], x3
        str             d3, [x0]

        ret
endfunc

// -----------------------------------------------------------------------------
// pred_planar_16x16_8: Planar prediction
//
// Formula: POS(x,y) = ((15-x)*left[y] + (x+1)*top[16]
//                     + (15-y)*top[x]  + (y+1)*left[16] + 16) >> 5
//
// Decomposed into incremental base update (same as 4x4/8x8/32x32):
//   base_0[x] = inc[x]*top[16] + 15*top[x] + left[16] + 16
//   base_{y+1}[x] = base_y[x] - (top[x] - left[16])
//   POS(x,y) = (base_y[x] + weights_dec[x]*left[y]) >> 5
//
// 16-wide requires split-half processing (umull/umull2).
// 16 rows fully unrolled with NEON-domain left[y] broadcast.
//
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// -----------------------------------------------------------------------------
function ff_hevc_pred_planar_16x16_8_neon, export=1
        // Load reference samples
        ldr             q0, [x1]                // top[0..15]
        ldr             q1, [x2]                // left[0..15]
        ldrb            w4, [x1, #16]           // top[16]
        ldrb            w5, [x2, #16]           // left[16]

        // Setup weight vectors for 16 elements
        movrel          x6, planar_weights_16
        ldp             q4, q5, [x6]            // weights_dec [15..0], weights_inc [1..16]

        // Precompute base[x] = inc[x]*top[16] + 15*top[x] + left[16]
        dup             v6.16b, w4              // top[16] broadcast
        umull           v19.8h, v5.8b, v6.8b    // inc[x]*top[16] lo
        umull2          v20.8h, v5.16b, v6.16b  // inc[x]*top[16] hi

        // 15*top[x] = (top[x]<<4) - top[x]
        uxtl            v2.8h, v0.8b            // widen top[0..7]
        uxtl2           v3.8h, v0.16b           // widen top[8..15]
        ushll           v6.8h, v0.8b, #4        // top[0..7]<<4
        ushll2          v7.8h, v0.16b, #4       // top[8..15]<<4
        sub             v6.8h, v6.8h, v2.8h     // 15*top[0..7]
        sub             v7.8h, v7.8h, v3.8h     // 15*top[8..15]

        add             v19.8h, v19.8h, v6.8h   // + 15*top lo
        add             v20.8h, v20.8h, v7.8h   // + 15*top hi

        dup             v5.8h, w5               // left[16] as 16-bit
        add             v19.8h, v19.8h, v5.8h   // + left[16]
        add             v20.8h, v20.8h, v5.8h
        // v19/v20 = base_lo/base_hi for row 0 (rounding folded into rshrn below)

        // Precompute decrement: top[x] - left[16]
        sub             v21.8h, v2.8h, v5.8h    // dec lo = top[0..7] - left[16]
        sub             v22.8h, v3.8h, v5.8h    // dec hi = top[8..15] - left[16]

        // Persistent registers:
        //   v19,v20 = base[0..15] (16-bit, decremented each row)
        //   v21,v22 = decrement[0..15] (16-bit, constant)
        //   v4      = weight_dec[0..15] (8-bit, constant)
        //   v1      = left[0..15] (8-bit, preloaded for NEON-domain broadcast)

.macro planar16_row lane, last=0
        dup             v6.16b, v1.b[\lane]     // left[y] NEON-domain broadcast
        umull           v16.8h, v4.8b, v6.8b    // dec[x]*left[y] lo
        umull2          v17.8h, v4.16b, v6.16b  // dec[x]*left[y] hi
        add             v16.8h, v16.8h, v19.8h  // + base lo
        add             v17.8h, v17.8h, v20.8h  // + base hi
        rshrn           v16.8b, v16.8h, #5      // >>5 lo (rounded)
        rshrn2          v16.16b, v17.8h, #5     // >>5 hi, merge into q16
  .if \last == 0
        st1             {v16.16b}, [x0], x3
        sub             v19.8h, v19.8h, v21.8h  // base -= decrement
        sub             v20.8h, v20.8h, v22.8h
  .else
        str             q16, [x0]
  .endif
.endm

        planar16_row    0
        planar16_row    1
        planar16_row    2
        planar16_row    3
        planar16_row    4
        planar16_row    5
        planar16_row    6
        planar16_row    7
        planar16_row    8
        planar16_row    9
        planar16_row    10
        planar16_row    11
        planar16_row    12
        planar16_row    13
        planar16_row    14
        planar16_row    15, last=1

.purgem planar16_row

        ret
endfunc

// -----------------------------------------------------------------------------
// pred_planar_32x32_8: Planar prediction
//
// Formula: POS(x,y) = ((31-x)*left[y] + (x+1)*top[32]
//                     + (31-y)*top[x]  + (y+1)*left[32] + 32) >> 6
//
// Decomposed as:  base[x] = weight_inc[x]*top[32] + 31*top[x] + 32
//                 Per row:  base[x] += left[32]    (incremental for (y+1)*left[32])
//                           base[x] -= top[x]      (incremental for (31-y)*top[x])
//                           result   = base[x] + weight_dec[x]*left[y]
//
// Both row_add and the (31-y)*top[x] term are folded into the base,
// eliminating all GP→NEON scalar broadcasts except for left[y].
// The loop processes 8 rows per iteration (macro-expanded), with 4
// iterations total. left[y] values are loaded 8 at a time and
// broadcast per-lane within each iteration.
//
// Arguments:
// x0: src
// x1: top
// x2: left
// x3: stride
// -----------------------------------------------------------------------------
function ff_hevc_pred_planar_32x32_8_neon, export=1
        // Load top[0..31]
        ldp             q0, q1, [x1]            // top[0..15], top[16..31]
        ldrb            w4, [x1, #32]           // top[32]
        ldrb            w5, [x2, #32]           // left[32]

        // Load weight vectors
        movrel          x6, planar_weights_32
        ldp             q4, q5, [x6]            // weight_dec = {31,30,...,0}
        ldp             q6, q7, [x6, #32]       // weight_inc = {1,2,...,32}

        // Precompute term_A = weight_inc * top[32]  (16-bit)
        dup             v2.16b, w4
        umull           v20.8h, v6.8b, v2.8b
        umull2          v21.8h, v6.16b, v2.16b
        umull           v22.8h, v7.8b, v2.8b
        umull2          v23.8h, v7.16b, v2.16b

        // Widen top[x] for incremental subtraction
        uxtl            v24.8h, v0.8b
        uxtl2           v25.8h, v0.16b

        // 31*top[x] = top[x]<<5 - top[x]
        ushll           v6.8h, v0.8b, #5
        ushll2          v7.8h, v0.16b, #5
        sub             v6.8h, v6.8h, v24.8h    // 31*top[0..7]
        sub             v7.8h, v7.8h, v25.8h    // 31*top[8..15]

        // base[0..15] = term_A + 31*top[0..15]
        add             v20.8h, v20.8h, v6.8h
        add             v21.8h, v21.8h, v7.8h

        // Same for top[16..31]
        uxtl            v26.8h, v1.8b
        uxtl2           v27.8h, v1.16b
        ushll           v6.8h, v1.8b, #5
        ushll2          v7.8h, v1.16b, #5
        sub             v6.8h, v6.8h, v26.8h     // 31*top[16..23]
        sub             v7.8h, v7.8h, v27.8h     // 31*top[24..31]
        add             v22.8h, v22.8h, v6.8h
        add             v23.8h, v23.8h, v7.8h

        // Compute combined decrement: top[x] - left[32]
        // Each row: base += left[32] and base -= top[x]
        // Combined: base -= (top[x] - left[32])
        dup             v3.8h, w5               // left[32] as 16-bit
        sub             v24.8h, v24.8h, v3.8h   // top[0..7] - left[32]
        sub             v25.8h, v25.8h, v3.8h   // top[8..15] - left[32]
        sub             v26.8h, v26.8h, v3.8h   // top[16..23] - left[32]
        sub             v27.8h, v27.8h, v3.8h   // top[24..31] - left[32]

        // Now base needs initial +=left[32] for y=0 (row_add = 1*left[32])
        add             v20.8h, v20.8h, v3.8h
        add             v21.8h, v21.8h, v3.8h
        add             v22.8h, v22.8h, v3.8h
        add             v23.8h, v23.8h, v3.8h

        // Persistent registers:
        //   v20-v23 = base[0..31] (includes running row_add, decremented by combined each row)
        //   v24,v25 = top[0..15] - left[32] (combined decrement)
        //   v26,v27 = top[16..31] - left[32] (combined decrement)
        //   v4,v5   = weight_dec[0..31] (8-bit)

.macro planar32_row lane, leftreg
        dup             v2.16b, \leftreg\().b[\lane]
        umull           v16.8h, v4.8b, v2.8b
        umull2          v17.8h, v4.16b, v2.16b
        umull           v18.8h, v5.8b, v2.8b
        umull2          v19.8h, v5.16b, v2.16b
        add             v16.8h, v16.8h, v20.8h
        add             v17.8h, v17.8h, v21.8h
        add             v18.8h, v18.8h, v22.8h
        add             v19.8h, v19.8h, v23.8h
        rshrn           v28.8b, v16.8h, #6
        rshrn2          v28.16b, v17.8h, #6
        rshrn           v29.8b, v18.8h, #6
        rshrn2          v29.16b, v19.8h, #6
        stp             q28, q29, [x0]
        add             x0, x0, x3
        sub             v20.8h, v20.8h, v24.8h
        sub             v21.8h, v21.8h, v25.8h
        sub             v22.8h, v22.8h, v26.8h
        sub             v23.8h, v23.8h, v27.8h
.endm

        // Process 32 rows in 4 iterations of 8 rows each
        mov             w7, #4
.Lplanar32_loop:
        ld1             {v1.8b}, [x2], #8       // load 8 left[] bytes for this iteration

        planar32_row    0, v1
        planar32_row    1, v1
        planar32_row    2, v1
        planar32_row    3, v1
        planar32_row    4, v1
        planar32_row    5, v1
        planar32_row    6, v1
        planar32_row    7, v1

        subs            w7, w7, #1
        b.gt            .Lplanar32_loop

.purgem planar32_row

        ret
endfunc


// =============================================================================
// Weight tables for planar prediction
// =============================================================================

const planar_weights_4, align=4
        .byte   3, 2, 1, 0, 0, 0, 0, 0          // weights_dec for 4x4
        .byte   1, 2, 3, 4, 0, 0, 0, 0          // weights_inc for 4x4
endconst

const planar_weights_8, align=4
        .byte   7, 6, 5, 4, 3, 2, 1, 0          // weights_dec
        .byte   1, 2, 3, 4, 5, 6, 7, 8          // weights_inc
endconst

const planar_weights_16, align=4
        .byte   15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
        .byte   1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
endconst

const planar_weights_32, align=4
        .byte   31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16
        .byte   15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
        .byte   1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
        .byte   17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32
endconst

// =============================================================================
// Reference Sample Filtering — 3-tap filter [1,2,1]>>2
// =============================================================================

/*
 * 3-tap filter: out[i] = (in[i-1] + 2*in[i] + in[i+1] + 2) >> 2
 *
 * Signature: void ff_hevc_ref_filter_3tap_NxN_8_neon(
 *     uint8_t *filtered_left,   // x0
 *     uint8_t *filtered_top,    // x1
 *     const uint8_t *left,      // x2
 *     const uint8_t *top,       // x3
 *     int size)                 // w4 — (unused, size is hardcoded per entry point)
 *
 * All pointers point to index [0]; caller has adjusted for [-1] accessibility.
 *
 * NEON approach: load 32 bytes via ld1, use ext #1/#2 to form
 *   prev(v0)/curr(v2)/next(v3). Fold the 3-tap kernel into two
 *   halving-adds on 16 bytes (no widen/narrow):
 *     uhadd  v4, v0, v3        // floor((prev + next) / 2)
 *     urhadd v4, v4, v2        // (h + curr + 1) / 2
 *   equivalent to (prev + 2*curr + next + 2) >> 2 for all u8 inputs.
 *   Caller pads input arrays by 16 bytes to guarantee safe 32-byte loads.
 *   Forward processing is safe since output goes to a separate buffer.
 */

// Load 32 bytes from [x12], compute 16 filtered samples into v4.16b
.macro filter_3tap_compute
        ld1             {v0.16b, v1.16b}, [x12]
        ext             v2.16b, v0.16b, v1.16b, #1 // curr
        ext             v3.16b, v0.16b, v1.16b, #2 // next
        uhadd           v4.16b, v0.16b, v3.16b   // floor((prev + next) / 2)
        urhadd          v4.16b, v4.16b, v2.16b   // (prev + 2*curr + next + 2) >> 2
.endm

// Filter one array. n_reg holds 2*size; filters n-1 samples, copies last.
// Caller guarantees 16 bytes of readable padding beyond in[n-1].
// For total >= 16, overlap-last-16 avoids partial stores.
// For total < 16 (8x8), overlap-last-8 stores two 8-byte halves.
.macro filter_3tap_array x_out, x_in, n_reg
        // Copy last element: out[n-1] = in[n-1]
        sub             x10, \n_reg, #1
        ldrb            w11, [\x_in, x10]
        strb            w11, [\x_out, x10]

        sub             x12, \x_in, #1          // x12 = load ptr (&in[-1])
        mov             x13, \x_out             // x13 = store ptr (&out[0])
        sub             x14, \n_reg, #1         // x14 = total samples to filter

        cmp             x14, #16
        b.lt            6f                      // total < 16: small path (8x8)

        // --- Main loop: 16 samples per iteration ---
1:      filter_3tap_compute
        st1             {v4.16b}, [x13], #16
        add             x12, x12, #16
        sub             x14, x14, #16
        cmp             x14, #16
        b.ge            1b

        // --- Overlap last 16 samples ---
        cbz             x14, 9f
        add             x12, \x_in, \n_reg
        sub             x12, x12, #18          // &in[n-18]
        add             x13, \x_out, \n_reg
        sub             x13, x13, #17          // &out[n-17]
        filter_3tap_compute
        st1             {v4.16b}, [x13]
        b               9f

        // --- Small path: total < 16 (8x8, total=15) ---
        // Store as two overlapping 8-byte halves: out[0..7] + out[7..14]
6:      filter_3tap_compute
        ext             v5.16b, v4.16b, v4.16b, #7
        str             d4, [x13]               // out[0..7]
        str             d5, [x13, #7]           // out[7..14]
9:
.endm

// Corner: filtered[-1] = (left[0] + 2*left[-1] + top[0] + 2) >> 2
.macro filter_3tap_corner
        ldrb            w10, [x2]               // left[0]
        ldrb            w11, [x2, #-1]          // left[-1]
        ldrb            w12, [x3]               // top[0]
        add             w10, w10, w12           // left[0] + top[0]
        add             w10, w10, w11, lsl #1   // + 2*left[-1]
        add             w10, w10, #2            // + 2
        lsr             w10, w10, #2            // >> 2
        strb            w10, [x0, #-1]          // filtered_left[-1]
        strb            w10, [x1, #-1]          // filtered_top[-1]
.endm

function ff_hevc_ref_filter_3tap_8x8_8_neon, export=1
        mov             x4, #16                 // n = 2*8
        b               hevc_ref_filter_3tap_common_8_neon
endfunc

function ff_hevc_ref_filter_3tap_16x16_8_neon, export=1
        mov             x4, #32                 // n = 2*16
        b               hevc_ref_filter_3tap_common_8_neon
endfunc

function ff_hevc_ref_filter_3tap_32x32_8_neon, export=1
        mov             x4, #64                 // n = 2*32
        b               hevc_ref_filter_3tap_common_8_neon
endfunc

function hevc_ref_filter_3tap_common_8_neon, export=0
        filter_3tap_array x0, x2, x4
        filter_3tap_corner
        filter_3tap_array x1, x3, x4
        ret
endfunc

.purgem filter_3tap_compute
.purgem filter_3tap_array
.purgem filter_3tap_corner

// =============================================================================
// Reference Sample Filtering — Strong Intra Smoothing
// =============================================================================

/*
 * Strong intra smoothing (32x32 luma only):
 *   filtered_top[i] = ((64-(i+1))*top[-1] + (i+1)*top[63] + 32) >> 6
 *   left[i]         = ((64-(i+1))*left[-1] + (i+1)*left[63] + 32) >> 6
 *   for i = 0..62
 *
 * Also sets: filtered_top[-1] = top[-1], filtered_top[63] = top[63]
 *
 * Signature: void ff_hevc_ref_filter_strong_8_neon(
 *     uint8_t *filtered_top,  // x0
 *     uint8_t *left,          // x1
 *     const uint8_t *top)     // x2
 *
 * NEON approach:
 *   Preloaded weights dec={63..1,0} in v0-v3, inc={1..63,0} in v4-v7.
 *   Two 16-byte blocks interleaved per pair, written via st1 {v22,v23},
 *   hiding the rshrn→st1 latency. Weight[63]=0 lets lane 15 fall out as
 *   zero; one strb restores output[63] to the correct end value.
 */

// Precomputed weight tables in .rodata
const filter_strong_weights_inc, align=4
        .byte    1,  2,  3,  4,  5,  6,  7,  8
        .byte    9, 10, 11, 12, 13, 14, 15, 16
        .byte   17, 18, 19, 20, 21, 22, 23, 24
        .byte   25, 26, 27, 28, 29, 30, 31, 32
        .byte   33, 34, 35, 36, 37, 38, 39, 40
        .byte   41, 42, 43, 44, 45, 46, 47, 48
        .byte   49, 50, 51, 52, 53, 54, 55, 56
        .byte   57, 58, 59, 60, 61, 62, 63,  0
endconst

const filter_strong_weights_dec, align=4
        .byte   63, 62, 61, 60, 59, 58, 57, 56
        .byte   55, 54, 53, 52, 51, 50, 49, 48
        .byte   47, 46, 45, 44, 43, 42, 41, 40
        .byte   39, 38, 37, 36, 35, 34, 33, 32
        .byte   31, 30, 29, 28, 27, 26, 25, 24
        .byte   23, 22, 21, 20, 19, 18, 17, 16
        .byte   15, 14, 13, 12, 11, 10,  9,  8
        .byte    7,  6,  5,  4,  3,  2,  1,  0
endconst

// Macro: apply strong smoothing to one 63-element array
// x_out = output pointer (index 0), w_start = start pixel, w_end = end pixel
// Weights must be preloaded: v0-v3 = dec weights, v4-v7 = inc weights (16b each)
// Interleave two 16-byte blocks to hide rshrn→st1 latency on in-order cores.
.macro strong_smooth x_out, w_start, w_end
        dup             v16.16b, \w_start       // broadcast start
        dup             v17.16b, \w_end         // broadcast end
        mov             x14, \x_out

        // Blocks 0+1 interleaved: indices 0-31
        umull           v18.8h, v0.8b, v16.8b   // blk0: dec_lo * start
        umull           v20.8h, v1.8b, v16.8b   // blk1: dec_lo * start
        umull2          v19.8h, v0.16b, v16.16b // blk0: dec_hi * start
        umull2          v21.8h, v1.16b, v16.16b // blk1: dec_hi * start
        umlal           v18.8h, v4.8b, v17.8b   // blk0: + inc_lo * end
        umlal           v20.8h, v5.8b, v17.8b   // blk1: + inc_lo * end
        umlal2          v19.8h, v4.16b, v17.16b // blk0: + inc_hi * end
        umlal2          v21.8h, v5.16b, v17.16b // blk1: + inc_hi * end
        rshrn           v22.8b, v18.8h, #6      // blk0: narrow lo
        rshrn           v23.8b, v20.8h, #6      // blk1: narrow lo
        rshrn2          v22.16b, v19.8h, #6     // blk0: narrow hi
        rshrn2          v23.16b, v21.8h, #6     // blk1: narrow hi
        st1             {v22.16b, v23.16b}, [x14], #32

        // Blocks 2+3 interleaved: indices 32-63
        // Weight at index 63 is zero, so v23.b[15] computes 0; we store all
        // 32 bytes and fix up output[63] with a scalar strb afterward.
        umull           v18.8h, v2.8b, v16.8b   // blk2: dec_lo * start
        umull           v20.8h, v3.8b, v16.8b   // blk3: dec_lo * start
        umull2          v19.8h, v2.16b, v16.16b // blk2: dec_hi * start
        umull2          v21.8h, v3.16b, v16.16b // blk3: dec_hi * start
        umlal           v18.8h, v6.8b, v17.8b   // blk2: + inc_lo * end
        umlal           v20.8h, v7.8b, v17.8b   // blk3: + inc_lo * end
        umlal2          v19.8h, v6.16b, v17.16b // blk2: + inc_hi * end
        umlal2          v21.8h, v7.16b, v17.16b // blk3: + inc_hi * end
        rshrn           v22.8b, v18.8h, #6      // blk2: narrow lo
        rshrn           v23.8b, v20.8h, #6      // blk3: narrow lo
        rshrn2          v22.16b, v19.8h, #6     // blk2: narrow hi
        rshrn2          v23.16b, v21.8h, #6     // blk3: narrow hi
        st1             {v22.16b, v23.16b}, [x14] // indices 32-63
        strb            \w_end, [x14, #31]      // restore output[63]
.endm

function ff_hevc_ref_filter_strong_8_neon, export=1
        ldrb            w10, [x2, #-1]          // top[-1]
        strb            w10, [x0, #-1]          // filtered_top[-1]
        ldrb            w11, [x2, #63]          // top[63]
        strb            w11, [x0, #63]          // filtered_top[63]

        // Preload all weight tables into registers (shared across both passes)
        movrel          x8, filter_strong_weights_dec
        movrel          x9, filter_strong_weights_inc
        ld1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x8]
        ld1             {v4.16b, v5.16b, v6.16b, v7.16b}, [x9]

        // Smooth top → filtered_top[0..62]
        strong_smooth   x0, w10, w11

        // Smooth left → left[0..62] in-place
        ldrb            w10, [x1, #-1]          // left[-1]
        ldrb            w11, [x1, #63]          // left[63]
        strong_smooth   x1, w10, w11

        ret
endfunc

.purgem strong_smooth

// =============================================================================
// Angular Prediction
// =============================================================================

// -----------------------------------------------------------------------------
// pred_angular_mode_10_8: Horizontal prediction (mode 10)
// Caller must ensure top[-1] and left[-1] are valid (used for edge smoothing
// when c_idx == 0 and size < 32).
// Arguments:
// x0: src
// x1: top (only used for edge smoothing)
// x2: left
// x3: stride
// w4: c_idx
// w5: log2_size
// -----------------------------------------------------------------------------
function ff_hevc_pred_angular_mode_10_8_neon, export=1
        cmp             w5, #3
        b.lt            .Lmode10_4x4
        b.eq            .Lmode10_8x8
        cmp             w5, #4
        b.eq            .Lmode10_16x16

        // --- size 32: 2 rows per iteration using ld2r ---
        mov             w7, #32
        add             x8, x0, x3              // x8 = row 1 pointer
        lsl             x9, x3, #1              // x9 = stride * 2
.Lmode10_32x32_row:
        ld2r            {v0.16b, v1.16b}, [x2], #2
        subs            w7, w7, #2
        stp             q0, q0, [x0]
        stp             q1, q1, [x8]
        add             x0, x0, x9
        add             x8, x8, x9
        b.gt            .Lmode10_32x32_row
        // size 32 never does edge smoothing
        ret

        // --- size 16: 2 rows per iteration using ld2r + dual pointer ---
.Lmode10_16x16:
        mov             x6, x0                  // save src base
        mov             x7, x2                  // save left base for edge smooth
        add             x8, x0, x3              // x8 = odd-row pointer
        lsl             x9, x3, #1              // x9 = stride * 2
        mov             w10, #16
.Lmode10_16x16_row:
        ld2r            {v0.16b, v1.16b}, [x2], #2
        subs            w10, w10, #2
        st1             {v0.16b}, [x0], x9
        st1             {v1.16b}, [x8], x9
        b.gt            .Lmode10_16x16_row
        mov             x2, x7                  // restore left base
        b               .Lmode10_edge_smooth

        // --- size 8: ld4r to load 4 rows at once ---
.Lmode10_8x8:
        mov             x6, x0                  // save src base
        mov             x7, x2                  // save left base for edge smooth
        add             x8, x0, x3              // x8 = odd-row pointer
        lsl             x9, x3, #1              // x9 = stride * 2
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], #4
        st1             {v0.8b}, [x0], x9
        st1             {v1.8b}, [x8], x9
        ld4r            {v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #4
        st1             {v2.8b}, [x0], x9
        st1             {v3.8b}, [x8], x9
        st1             {v4.8b}, [x0], x9
        st1             {v5.8b}, [x8], x9
        st1             {v6.8b}, [x0]
        st1             {v7.8b}, [x8]
        mov             x2, x7                  // restore left base
        b               .Lmode10_edge_smooth

        // --- size 4: ld4r to load all 4 rows at once ---
.Lmode10_4x4:
        mov             x6, x0                  // save src base
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b}, [x2]
        str             s0, [x0]
        str             s1, [x0, x3]
        add             x0, x0, x3, lsl #1
        str             s2, [x0]
        str             s3, [x0, x3]

.Lmode10_edge_smooth:
        cbnz            w4, .Lmode10_ret

        sub             x7, x1, #1              // top - 1 (hoisted early)
        mov             x0, x6                  // restore src base

        ld1r            {v5.16b}, [x2]          // left[0] broadcast
        ld1r            {v1.16b}, [x7]          // top[-1] broadcast

        cmp             w5, #3
        b.lt            .Lmode10_smooth_4
        b.eq            .Lmode10_smooth_8

        // size 16 edge smoothing: out[x] = clip8(left[0] + (top[x] - top[-1]) / 2)
        ldr             q2, [x1]                // top[0..15]
        uhsub           v2.16b, v2.16b, v1.16b  // signed half-difference
        usqadd          v5.16b, v2.16b          // sat_u8(left[0] + signed_delta)
        st1             {v5.16b}, [x0]
        ret

.Lmode10_smooth_4:
        ldr             s2, [x1]                // top[0..3]
        uhsub           v2.8b, v2.8b, v1.8b
        usqadd          v5.8b, v2.8b
        st1             {v5.s}[0], [x0]
        ret

.Lmode10_smooth_8:
        ldr             d2, [x1]                // top[0..7]
        uhsub           v2.8b, v2.8b, v1.8b
        usqadd          v5.8b, v2.8b
        st1             {v5.8b}, [x0]

.Lmode10_ret:
        ret
endfunc

// -----------------------------------------------------------------------------
// pred_angular_mode_26_8: Vertical prediction (mode 26)
// Caller must ensure top[-1] and left[-1] are valid (used for edge smoothing
// when c_idx == 0 and size < 32).
// Arguments:
// x0: src
// x1: top
// x2: left (only used for edge smoothing)
// x3: stride
// w4: c_idx
// w5: log2_size
// -----------------------------------------------------------------------------
function ff_hevc_pred_angular_mode_26_8_neon, export=1
        mov             x7, x0                  // x7 = write pointer (preserve x0)

        cmp             w5, #3
        b.lt            .Lmode26_4x4
        b.eq            .Lmode26_8x8
        cmp             w5, #4
        b.eq            .Lmode26_16x16
        // fall-through to 32x32

        // --- size 32 ---
        ldp             q0, q1, [x1]            // Load top[0..31] once
        mov             w9, #32
.Lmode26_32x32_row:
        subs            w9, w9, #1
        st1             {v0.16b, v1.16b}, [x7], x3
        b.gt            .Lmode26_32x32_row
        b               .Lmode26_edge_smooth

        // --- size 16 ---
.Lmode26_16x16:
        ldr             q0, [x1]                // Load top[0..15] once
        mov             w9, #16
.Lmode26_16x16_row:
        subs            w9, w9, #1
        st1             {v0.16b}, [x7], x3
        b.gt            .Lmode26_16x16_row
        b               .Lmode26_edge_smooth

        // --- size 8 ---
.Lmode26_8x8:
        ldr             d0, [x1]                // Load top[0..7] once
        mov             w9, #8
.Lmode26_8x8_row:
        subs            w9, w9, #1
        st1             {v0.8b}, [x7], x3
        b.gt            .Lmode26_8x8_row
        b               .Lmode26_edge_smooth

        // --- size 4 ---
.Lmode26_4x4:
        ldr             s0, [x1]                // Load top[0..3] once
        mov             w9, #4
.Lmode26_4x4_row:
        subs            w9, w9, #1
        str             s0, [x7]
        add             x7, x7, x3
        b.gt            .Lmode26_4x4_row

.Lmode26_edge_smooth:
        cbnz            w4, .Lmode26_ret
        cmp             w5, #5
        b.ge            .Lmode26_ret

        // Edge smoothing: out[y] = clip8(top[0] + (left[y] - left[-1]) / 2)
        ld1r            {v5.16b}, [x1]          // top[0] broadcast
        sub             x8, x2, #1
        ld1r            {v1.16b}, [x8]          // left[-1] broadcast

        cmp             w5, #3
        b.lt            .Lmode26_smooth_4
        b.eq            .Lmode26_smooth_8

        // size 16
        ldr             q2, [x2]                // left[0..15]
        uhsub           v2.16b, v2.16b, v1.16b  // signed half-difference
        usqadd          v5.16b, v2.16b          // sat_u8(top[0] + signed_delta)

        // Store smoothed column[0] for 16 rows using precomputed addresses
        // Reordered to avoid direct dependency chains
        add             x10, x0, x3, lsl #1     // x10 = row 2
        add             x9, x0, x3              // x9 = row 1
        add             x11, x10, x3            // x11 = row 3
        st1             {v5.b}[0], [x0]
        add             x0, x10, x3, lsl #1     // x0 = row 4 (after last use of old x0)
        st1             {v5.b}[1], [x9]
        st1             {v5.b}[2], [x10]
        st1             {v5.b}[3], [x11]
        add             x10, x0, x3, lsl #1     // x10 = row 6
        add             x9, x0, x3              // x9 = row 5
        add             x11, x10, x3            // x11 = row 7
        st1             {v5.b}[4], [x0]
        add             x0, x10, x3, lsl #1     // x0 = row 8
        st1             {v5.b}[5], [x9]
        st1             {v5.b}[6], [x10]
        st1             {v5.b}[7], [x11]
        add             x10, x0, x3, lsl #1     // x10 = row 10
        add             x9, x0, x3              // x9 = row 9
        add             x11, x10, x3            // x11 = row 11
        st1             {v5.b}[8], [x0]
        add             x0, x10, x3, lsl #1     // x0 = row 12
        st1             {v5.b}[9], [x9]
        st1             {v5.b}[10], [x10]
        st1             {v5.b}[11], [x11]
        add             x10, x0, x3, lsl #1     // x10 = row 14
        add             x9, x0, x3              // x9 = row 13
        add             x11, x10, x3            // x11 = row 15
        st1             {v5.b}[12], [x0]
        st1             {v5.b}[13], [x9]
        st1             {v5.b}[14], [x10]
        st1             {v5.b}[15], [x11]
        b               .Lmode26_ret

.Lmode26_smooth_4:
        ldr             s2, [x2]                // left[0..3]
        uhsub           v2.8b, v2.8b, v1.8b
        usqadd          v5.8b, v2.8b
        add             x10, x0, x3, lsl #1
        add             x9, x0, x3
        add             x11, x10, x3
        st1             {v5.b}[0], [x0]
        st1             {v5.b}[1], [x9]
        st1             {v5.b}[2], [x10]
        st1             {v5.b}[3], [x11]
        b               .Lmode26_ret

.Lmode26_smooth_8:
        ldr             d2, [x2]                // left[0..7]
        uhsub           v2.8b, v2.8b, v1.8b
        usqadd          v5.8b, v2.8b
        add             x10, x0, x3, lsl #1     // x10 = row 2
        add             x9, x0, x3              // x9 = row 1
        add             x11, x10, x3            // x11 = row 3
        st1             {v5.b}[0], [x0]
        add             x0, x10, x3, lsl #1     // x0 = row 4
        st1             {v5.b}[1], [x9]
        st1             {v5.b}[2], [x10]
        st1             {v5.b}[3], [x11]
        add             x10, x0, x3, lsl #1
        add             x9, x0, x3
        add             x11, x10, x3
        st1             {v5.b}[4], [x0]
        st1             {v5.b}[5], [x9]
        st1             {v5.b}[6], [x10]
        st1             {v5.b}[7], [x11]
        b               .Lmode26_ret

.Lmode26_ret:
        ret
endfunc
