/* Copyright (c) 2026 Lynne */ /* Ported and relicensed from FFmpeg, libavutil/aarch64/tx_float_neon.S */ /* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * AArch64 NEON inverse MDCT for CELT, ported from FFmpeg's * libavutil/aarch64/tx_float_neon.S (commit d229b4c1242870) * * The code is deliberately kept STRUCTURALLY IDENTICAL to FFmpeg's so * that future updates to that file can be applied here mechanically. * The transform tree for the CELT inverse MDCT sizes N/2 = 15*M*2, * M = {4, 8, 16, 32, 64}, is * * celt_tx_mdct_inv_float_neon mirrors ff_tx_mdct_inv_float_neon * -> celt_tx_fft_pfa_15xM_ns_float_neon ff_tx_fft_pfa_15xM_ns_float_neon * -> celt_tx_fft4_fwd_float_neon ff_tx_fft4_fwd_float_neon * celt_tx_fft8_ns_float_neon ff_tx_fft8_ns_float_neon * celt_tx_fft16_ns_float_neon ff_tx_fft16_ns_float_neon * celt_tx_fft32_ns_float_neon ff_tx_fft32_ns_float_neon * celt_tx_fft_sr_ns_float_neon ff_tx_fft_sr_ns_float_neon * * driven by small AVTXContext-compatible structs and pre-baked tables in * celt_mdct_tx.c / celt_tx_tables.h instead of FFmpeg's runtime av_tx_init(). * Lines that differ from FFmpeg are flagged with `LIBOPUS DELTA`. The * classes of delta are: * * (P) Prefix. Functions are renamed ff_tx_* -> celt_tx_* so that libopus * can be linked into the same image as libavutil. endfunc/endconst * take the symbol name (celt_arm_asm.h needs it for ELF .size). * * (T) Tables. FFmpeg initializes ff_tx_tab_{53,32,64}_float at runtime * (ff_tx_init_tabs_float); here they are pre-baked, bit-exact, as * local consts, so X(ff_tx_tab_*) external references become local * celt_tx_tab_* ones. * * (I) Instantiations. Only the codelets reachable from the CELT inverse * MDCT (the preshuffled variants listed above) are emitted. FFmpeg * additionally emits fft2, fft4_inv, fft15 and the runtime * lookup-table (non-_ns) variants of every codelet; the shared * macros are kept verbatim, including their no_perm=0 paths. * * (S) Split-radix trim. CELT's largest sub-FFT under the MDCT is 512 * (power-of-two custom modes; the 15*M chain tops out at 64), so the * len >= 1024 recombination block and the SR_TRANSFORM_DEF chain * (which reference ff_tx_tab_1024..131072_float) are removed, as is * cmp_imm (only used by that chain). 512 is then always the final * length, so its stage falls straight into the deinterleave. * * (F) Frame records. The PFA and MDCT drivers (this port's own code) * spill x29/x30, set x29 = sp for frame-pointer unwinders and sign LR * under pac-ret; celt_arm_asm.h adds the BTI landing pads. * * Calling convention (same as FFmpeg's av_tx_fn): * x0 = const OpusTXContext *s (field offsets: len@0, map@8, exp@16, * tmp@24, sub@32, fn@40 -- LP64) * x1 = void *out * x2 = void *in * x3 = ptrdiff_t stride (bytes) */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #if defined(__aarch64__) /* Included outside the FIXED_POINT guard: the .note.gnu.property it emits must be present even when the kernels are compiled out, or the linker drops BTI/PAC for the whole library. */ #include "celt_arm_asm.h" #if !defined(FIXED_POINT) /* Open `doc/transforms.md` (FFmpeg) to see the code upon which the * transforms here were based upon. * * File conventions: * GPRs: x0-x3 - arguments, untouched * x4 - Lookup table base pointer * x5-x6 - macro ld1 temps/function scratch * x7-x9 - FFT table state * x10-x17 - lookup table/macro scratch * w19-w20 - current/target length when needed * x21-x22 - len*2, len*6 * * Vectors: v0-v7 - coefficients * v8-v15 - coefficients when needed, otherwise untouched * v16-v30 - used as needed * v31 - -1.0, +1.0, -1.0, +1.0. Never touched after loading. * * Stack: backup for v8-v15 and x19-x22 when needed, and transform lengths */ #define M_SQRT1_2 0.707106781186547524401 #define COS16_1 0.92387950420379638671875 #define COS16_3 0.3826834261417388916015625 /* We only ever load this once at the start, and then live with losing an * entire register as we need to lug this all the time everywhere. * Clearly should be integrated into an fsadd and fmlsa, but "muh RISC!". */ const subadd, align=4 .float -1.0, 1.0, -1.0, 1.0 endconst subadd .macro LOAD_SUBADD movrel x5, subadd ld1 { v31.4s }, [x5] .endm .macro SETUP_LUT no_lut=0 .if \no_lut == 0 ldr x4, [x0, #8] .endif .endm .macro LOAD_INPUT dst1, dst2, dst3, dst4, src, no_lut=0, discont=0 .if \no_lut == 1 .if \discont == 1 ldp q\dst1\(), q\dst2\(), [\src\()] ldp q\dst3\(), q\dst4\(), [\src\(), #32] add \src\(), \src\(), #64 .else ld1 { v\dst1\().4s, v\dst2\().4s, v\dst3\().4s, v\dst4\().4s }, [\src], #64 .endif .else ldp w10, w11, [x4, #0 ] ldp w12, w13, [x4, #8 ] ldp w14, w15, [x4, #16] ldp w16, w17, [x4, #24] add x4, x4, #32 ldr d\dst1, [\src, x10, lsl #3] add x11, \src, x11, lsl #3 ldr d\dst2, [\src, x12, lsl #3] add x13, \src, x13, lsl #3 ldr d\dst3, [\src, x14, lsl #3] add x15, \src, x15, lsl #3 ldr d\dst4, [\src, x16, lsl #3] add x17, \src, x17, lsl #3 ld1 { v\dst1\().d }[1], [x11] ld1 { v\dst2\().d }[1], [x13] ld1 { v\dst3\().d }[1], [x15] ld1 { v\dst4\().d }[1], [x17] .endif .endm .macro FFT4 e0, o0, standalone fadd v16.4s, \e0\().4s, \o0\().4s // r1..4 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1..4 rev64 v18.4s, \e0\().4s zip2 \o0\().2d, v16.2d, \e0\().2d zip1 v17.2d, v16.2d, \e0\().2d mov \o0\().d[1], v18.d[1] fadd \e0\().4s, v17.4s, \o0\().4s // a1,2 b1,4 fsub v16.4s, v17.4s, \o0\().4s // a3,4 b3,2 mov \o0\().16b, v16.16b // Swap once again... mov \o0\().s[3], \e0\().s[3] mov \e0\().s[3], v16.s[3] .if \standalone == 0 uzp2 \o0\().2d, \e0\().2d, \o0\().2d uzp1 \e0\().2d, \e0\().2d, v16.2d .endif .endm // Identical to FFT4, but does 2 transforms in parallel, with no deinterleaving .macro FFT4_X2 e0, o0, e1, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 fadd \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \t2\().4s, \e1\().4s, \o1\().4s // r1234 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1234 fsub \e1\().4s, \e1\().4s, \o1\().4s // t1234 rev64 \t4\().4s, \e0\().4s rev64 \t5\().4s, \e1\().4s zip2 \o0\().2d, \t0\().2d, \t4\().2d // t3,4 r3,4 zip2 \o1\().2d, \t2\().2d, \t5\().2d // t3,4 r3,4 fneg \t3\().4s, \t4\().4s fneg \t4\().4s, \t5\().4s mov \o0\().s[3], \t3\().s[3] mov \o1\().s[3], \t4\().s[3] zip1 \t1\().2d, \t0\().2d, \e0\().2d // t1,2 r1,2 zip1 \t3\().2d, \t2\().2d, \e1\().2d // t1,2 r1,2 fsub \t4\().4s, \t1\().4s, \o0\().4s // a34 b32 fadd \t5\().4s, \t1\().4s, \o0\().4s // a12 b14 fsub \t2\().4s, \t3\().4s, \o1\().4s // a34 b32 fadd \t3\().4s, \t3\().4s, \o1\().4s // a12 b14 zip1 \e0\().2d, \t5\().2d, \t4\().2d // a1234 zip2 \o0\().2d, \t5\().2d, \t4\().2d // b1432 zip1 \e1\().2d, \t3\().2d, \t2\().2d // a1234 zip2 \o1\().2d, \t3\().2d, \t2\().2d // b1432 .endm const tab_8pt, align=4 .float M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2 endconst tab_8pt .macro FFT8 e0, e1, o0, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 movrel x5, tab_8pt fsub \t1\().4s, \e1\().4s, \o1\().4s // j1234 fadd \o1\().4s, \e1\().4s, \o1\().4s // k1234 fsub \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \o0\().4s, \e0\().4s, \o0\().4s // q1234 ld1 { \t5\().4s }, [x5] ext \t4\().16b, \o1\().16b, \o1\().16b, #12 rev64 \t4\().4s, \t4\().4s ext \t2\().16b, \o0\().16b, \t4\().16b, #8 // o0[0,1], o1[3,2] mov \o0\().d[1], \t4\().d[1] // o0[3, 4]; o1[1, 4] fsub \e1\().4s, \o0\().4s, \t2\().4s // s34, g43 fadd \t2\().4s, \o0\().4s, \t2\().4s // s12, g12 rev64 \t6\().4s, v31.4s // 1, -1, 1, -1 dup \o0\().2d, \t0\().d[0] // r1212 dup \o1\().2d, \t0\().d[1] // r3434 rev64 \t4\().4s, \e1\().4s // xxg34 rev64 \o1\().4s, \o1\().4s // r4343 ext \t6\().16b, v31.16b, \t6\().16b, #8 // -1, 1, 1, -1 zip1 \t3\().2d, \t2\().2d, \e1\().2d // s1234 zip2 \t2\().2d, \t2\().2d, \t4\().2d // g1234 fadd \e0\().4s, \t3\().4s, \t2\().4s // out_e1 fsub \e1\().4s, \t3\().4s, \t2\().4s // out_e2 fmul \t1\().4s, \t1\().4s, \t5\().4s // j * +--+M_SQRT1_2 fmls \o0\().4s, \o1\().4s, \t6\().4s // z1234 rev64 \t4\().4s, \t1\().4s // j2143 fmla \t1\().4s, \t4\().4s, v31.4s // l2143 rev64 \t4\().4s, \t1\().4s // l1234 ext \t4\().16b, \t4\().16b, \t4\().16b, #8 // l3412 fmla \t4\().4s, \t1\().4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, \t4\().4s // out_o2 fsub \o0\().4s, \o0\().4s, \t4\().4s // out_o1 .endm // Identical as FFT8, but does 2 transforms in parallel .macro FFT8_X2 e0, e1, o0, o1, e2, e3, o2, o3 movrel x5, tab_8pt fadd v19.4s, \e3\().4s, \o3\().4s // k1234 fadd v17.4s, \e1\().4s, \o1\().4s // k1234 fadd v18.4s, \e2\().4s, \o2\().4s // q1234 fadd v16.4s, \e0\().4s, \o0\().4s // q1234 ld1 { v23.4s }, [x5] ext v22.16b, v19.16b, v19.16b, #12 ext v21.16b, v17.16b, v17.16b, #12 rev64 v22.4s, v22.4s rev64 v21.4s, v21.4s ext v19.16b, v18.16b, v22.16b, #8 ext v17.16b, v16.16b, v21.16b, #8 mov v18.d[1], v22.d[1] mov v21.d[0], v16.d[0] fadd v22.4s, v18.4s, v19.4s // s12, g12 fsub v19.4s, v18.4s, v19.4s // s34, g43 fsub v18.4s, v21.4s, v17.4s // s34, g43 fadd v16.4s, v21.4s, v17.4s // s12, g12 fsub \e0\().4s, \e0\().4s, \o0\().4s // r1234 fsub v20.4s, \e1\().4s, \o1\().4s // j1234 fsub \e2\().4s, \e2\().4s, \o2\().4s // r1234 fsub v21.4s, \e3\().4s, \o3\().4s // j1234 rev64 v24.4s, v31.4s // 1, -1, 1, -1 zip1 v17.2d, v16.2d, v18.2d // s1234 zip1 \e1\().2d, v22.2d, v19.2d // s1234 rev64 v18.4s, v18.4s // xxg34 rev64 v19.4s, v19.4s // xxg34 zip2 v16.2d, v16.2d, v18.2d // g1234 zip2 \e3\().2d, v22.2d, v19.2d // g1234 dup \o0\().2d, \e0\().d[0] // r1212 dup \o1\().2d, \e0\().d[1] // r3434 dup \o2\().2d, \e2\().d[0] // r1212 dup \o3\().2d, \e2\().d[1] // r3434 fadd \e2\().4s, \e1\().4s, \e3\().4s // out_e1 fsub \e3\().4s, \e1\().4s, \e3\().4s // out_e2 fadd \e0\().4s, v17.4s, v16.4s // out_e1 fsub \e1\().4s, v17.4s, v16.4s // out_e2 ext v24.16b, v31.16b, v24.16b, #8 // -1, 1, 1, -1 rev64 \o1\().4s, \o1\().4s // r4343 rev64 \o3\().4s, \o3\().4s // r4343 fmul v19.4s, v20.4s, v23.4s // j * +--+M_SQRT1_2 fmul v21.4s, v21.4s, v23.4s // j * +--+M_SQRT1_2 rev64 v20.4s, v19.4s // j2143 rev64 v18.4s, v21.4s // j2143 fmls \o0\().4s, \o1\().4s, v24.4s // z1234 fmls \o2\().4s, \o3\().4s, v24.4s // z1234 fmla v19.4s, v20.4s, v31.4s // l2143 fmla v21.4s, v18.4s, v31.4s // l2143 rev64 v20.4s, v19.4s // l1234 rev64 v18.4s, v21.4s // l1234 ext v20.16b, v20.16b, v20.16b, #8 // l3412 ext v18.16b, v18.16b, v18.16b, #8 // l3412 fmla v20.4s, v19.4s, v31.4s // t1234 fmla v18.4s, v21.4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, v20.4s // out_o2 fadd \o3\().4s, \o2\().4s, v18.4s // out_o2 fsub \o0\().4s, \o0\().4s, v20.4s // out_o1 fsub \o2\().4s, \o2\().4s, v18.4s // out_o1 .endm const tab_16pt, align=4 .float -COS16_1, COS16_1, -COS16_3, COS16_3 // Could be +-+- too .float COS16_3, COS16_3, COS16_1, COS16_1 .float 1.0, 1.0, M_SQRT1_2, M_SQRT1_2 endconst tab_16pt // 16-point FFT // t3, t4, t5, t6 must be sequential .macro FFT16 e0, e1, e2, e3, o0, o1, o2, o3, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 FFT8 \e0, \e1, \e2, \e3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 FFT4_X2 \o0, \o1, \o2, \o3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 movrel x5, tab_16pt rev64 \t0\().4s, \o0\().4s // z[ 8, 9].imre rev64 \t1\().4s, \o2\().4s // z[10,11].imre ins \t0\().d[0], xzr ins \t1\().d[0], xzr ld1 { \t4\().4s, \t5\().4s, \t6\().4s }, [x5] // TODO: We could derive \t4\() or \t5\() from either, but it seems cheaper to load fmla \o2\().4s, \t1\().4s, v31.4s // s[4567] fmls \o0\().4s, \t0\().4s, v31.4s // s[0123] fmul \t2\().4s, \o1\().4s, \t4\().4s fmul \t3\().4s, \o3\().4s, \t4\().4s rev64 \o3\().4s, \o3\().4s rev64 \o1\().4s, \o1\().4s fmla \t3\().4s, \o3\().4s, \t5\().4s // s[12, 13, 14, 15] fmls \t2\().4s, \o1\().4s, \t5\().4s // s[ 8, 9, 10, 11] fmul \t1\().4s, \o2\().4s, \t6\().4s // s[4567] * mult fmul \t0\().4s, \o0\().4s, \t6\().4s // s[0123] * mult mov \o1\().16b, \t3\().16b mov \o2\().16b, \t1\().16b fsub \t3\().4s, \t3\().4s, \t2\().4s // y34, u34 fsub \t1\().4s, \t1\().4s, \t0\().4s // w34, x34 fadd \t2\().4s, \t2\().4s, \o1\().4s // y56, u56 rev64 \t3\().4s, \t3\().4s fadd \t0\().4s, \t0\().4s, \o2\().4s // w56, x56 rev64 \t1\().4s, \t1\().4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fadd \o3\().4s, \e3\().4s, \t3\().4s fsub \o2\().4s, \e3\().4s, \t3\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o0\().4s, \e2\().4s, \t2\().4s fsub \e2\().4s, \e0\().4s, \t0\().4s fadd \e0\().4s, \e0\().4s, \t0\().4s fsub \e3\().4s, \e1\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t1\().4s .endm // LIBOPUS DELTA (I): ff_tx_fft2_float_neon is not instantiated (unreachable // from the CELT inverse MDCT sizes). .macro FFT4_FN name, inv function celt_tx_fft4_\name\()_float_neon, export=1 // LIBOPUS DELTA (P): ff_tx_ -> celt_tx_ ld1 {v0.4s, v1.4s}, [x2] .if \inv == 1 mov v2.d[0], v0.d[1] mov v0.d[1], v1.d[1] mov v1.d[1], v2.d[0] .endif FFT4 v0, v1, 1 st1 { v0.4s, v1.4s }, [x1] ret endfunc celt_tx_fft4_\name\()_float_neon .endm FFT4_FN fwd, 0 // LIBOPUS DELTA (I): FFT4_FN inv not instantiated (the preshuffled chain // encodes the direction in the maps and always uses the fwd kernel). .macro FFT8_FN name, no_perm function celt_tx_fft8_\name\()_neon, export=1 // LIBOPUS DELTA (P) SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_SUBADD FFT8 v0, v1, v2, v3 zip1 v16.2d, v0.2d, v2.2d zip2 v17.2d, v0.2d, v2.2d zip1 v18.2d, v1.2d, v3.2d zip2 v19.2d, v1.2d, v3.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1] ret endfunc celt_tx_fft8_\name\()_neon .endm FFT8_FN ns_float, 1 // LIBOPUS DELTA (I): FFT8_FN float (runtime-LUT variant) not instantiated. .macro FFT16_FN name, no_perm function celt_tx_fft16_\name\()_neon, export=1 // LIBOPUS DELTA (P) SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_SUBADD FFT16 v0, v1, v2, v3, v4, v5, v6, v7 zip1 v20.2d, v0.2d, v4.2d zip2 v21.2d, v0.2d, v4.2d zip1 v22.2d, v1.2d, v6.2d zip2 v23.2d, v1.2d, v6.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v2.2d, v5.2d zip2 v25.2d, v2.2d, v5.2d zip1 v26.2d, v3.2d, v7.2d zip2 v27.2d, v3.2d, v7.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1] ret endfunc celt_tx_fft16_\name\()_neon .endm FFT16_FN ns_float, 1 // LIBOPUS DELTA (I): FFT16_FN float not instantiated. const tab_15pt, align=4 .float 1.0, 1.0, -1.0, -1.0 endconst tab_15pt /* Tab_53 twiddles (v28..v30) duplicated/pre-signed once, instead of per * transform: v8/v9 = -+tab[8,9]/[10,11], v25/v28/v29 = tab[0,1]/[2,3]/[4,5], * v10 = +-tab[6,7]. v30 keeps tab[8..11]. Callers preserve d8-d10. */ .macro FFT15_DERIVE_CONSTS dup v8.2d, v30.d[0] dup v9.2d, v30.d[1] dup v10.2d, v29.d[1] dup v25.2d, v28.d[0] dup v28.2d, v28.d[1] dup v29.2d, v29.d[0] fmul v8.4s, v8.4s, v31.4s fmul v9.4s, v9.4s, v31.4s fmul v10.4s, v10.4s, v24.4s .endm .macro FFT15_LOAD no_perm, advance=0 .if \no_perm == 1 // Writebacks leave x2 a whole transform (120B) ahead for the PFA loop ld1 { v0.4s }, [x2], #16 // in[0,1] ld1r { v1.2d }, [x2], #8 // in[2] duplicated ld1 { v2.4s, v3.4s, v4.4s }, [x2], #48 // in[3..8] ld1 { v5.4s, v6.4s, v7.4s }, [x2], #48 // in[9..14] .else ldp w10, w11, [x4] // lut[0,1] ldr w12, [x4, #8] // lut[2] ldp w13, w14, [x4, #12] // lut[3,4] ldp w15, w16, [x4, #20] // lut[5,6] ldr d0, [x2, x10, lsl #3] add x10, x2, x11, lsl #3 add x12, x2, x12, lsl #3 ld1 { v0.d }[1], [x10] ld1r { v1.2d }, [x12] ldr d2, [x2, x13, lsl #3] add x13, x2, x14, lsl #3 ldr d3, [x2, x15, lsl #3] add x15, x2, x16, lsl #3 ld1 { v2.d }[1], [x13] ld1 { v3.d }[1], [x15] ldp w10, w11, [x4, #28] // lut[7,8] ldp w12, w13, [x4, #36] // lut[9,10] ldp w14, w15, [x4, #44] // lut[11,12] ldp w16, w17, [x4, #52] // lut[13,14] .if \advance == 1 add x4, x4, #60 .endif ldr d4, [x2, x10, lsl #3] add x10, x2, x11, lsl #3 ldr d5, [x2, x12, lsl #3] add x12, x2, x13, lsl #3 ldr d6, [x2, x14, lsl #3] add x14, x2, x15, lsl #3 ldr d7, [x2, x16, lsl #3] add x16, x2, x17, lsl #3 ld1 { v4.d }[1], [x10] ld1 { v5.d }[1], [x12] ld1 { v6.d }[1], [x14] ld1 { v7.d }[1], [x16] .endif .endm // Single 15-point FFT (see doc/transforms.md and the AVX2 FFT15); each ymm // becomes a pair of quads holding 2 complex each. Uses the derived constants // and tab_15pt in v24 (dc fold sign); with hoist_strides=1 the caller // provides x6/x7 = stride*3/*5. .macro FFT15_CORE hoist_strides=0 .if \hoist_strides == 0 add x6, x3, x3, lsl #1 // stride*3 add x7, x3, x3, lsl #2 // stride*5 .endif add x8, x1, x7 // &out[5] add x9, x8, x7 // &out[10] // 4x parallel 3pt over in[3..14] (the in[11..14] -+ signs are folded // into the twiddles: k = in[11..14] + Q4 -+ Q0), interleaved with the // dc 3pt over in[0..2] ([dc] tagged, v0 = dc[0] dup, v1 = dc[1,2]) fsub v16.4s, v2.4s, v4.4s // q[0,1]raw = in[3,4]-in[7,8] ext v26.16b, v0.16b, v0.16b, #8 // [dc] (in1, in0) fsub v17.4s, v3.4s, v5.4s // q[2,3]raw = in[5,6]-in[9,10] fadd v27.4s, v0.4s, v26.4s // [dc] pc[1]raw = in0+in1 fadd v2.4s, v2.4s, v4.4s // q[4,5]raw fsub v20.4s, v0.4s, v26.4s // [dc] (in0-in1, in1-in0) fadd v3.4s, v3.4s, v5.4s // q[6,7]raw rev64 v20.4s, v20.4s // [dc] pc[0]raw in hi half rev64 v16.4s, v16.4s // q[0,1]raw re/im-swapped ext v21.16b, v20.16b, v27.16b, #8 // [dc] (pc[0], pc[1]) rev64 v17.4s, v17.4s // q[2,3]raw re/im-swapped fadd v0.4s, v1.4s, v27.4s // [dc] dc[0] = in2 + pc[1]raw (dup) fadd v22.4s, v6.4s, v2.4s // y[0,1] = in[11,12] + q[4,5] fmul v21.4s, v21.4s, v30.4s // [dc] pc[0,1] scaled by tab[8..11] fadd v23.4s, v7.4s, v3.4s // y[2,3] = in[13,14] + q[6,7] fmul v16.4s, v16.4s, v8.4s // Q0[0,1] ext v26.16b, v21.16b, v21.16b, #8 // [dc] (pc[1], pc[0]) fmul v17.4s, v17.4s, v8.4s // Q0[2,3] fmla v21.4s, v26.4s, v24.4s // [dc] (dc[1]_int, dc[2]_int) fmla v6.4s, v2.4s, v9.4s // M[0,1] = in[11,12] + q*Q4mult fmla v7.4s, v3.4s, v9.4s // M[2,3] = in[13,14] + q*Q4mult fmla v1.4s, v21.4s, v31.4s // [dc] v1 = (dc[1], dc[2]) — DC done fsub v4.4s, v6.4s, v16.4s // k[0,1] = M[0,1] - Q0[0,1] fsub v5.4s, v7.4s, v17.4s // k[2,3] = M[2,3] - Q0[2,3] fadd v2.4s, v6.4s, v16.4s // k[4,5] = M[0,1] + Q0[0,1] fadd v3.4s, v7.4s, v17.4s // k[6,7] = M[2,3] + Q0[2,3] // 4pt butterflies on y (v22,v23), k[0..3] (v4,v5), k[4..7] (v2,v3); // one shared swapped operand per pair leaves the hi t's half-swapped, // which the dup-symmetric twiddles absorb and the output stage uses ext v16.16b, v23.16b, v23.16b, #8 // (y3, y2) ext v17.16b, v5.16b, v5.16b, #8 // (k3, k2) ext v20.16b, v3.16b, v3.16b, #8 // (k7, k6) fsub v21.4s, v22.4s, v16.4s // (t3, t2) fadd v22.4s, v22.4s, v16.4s // (t0, t1) fsub v26.4s, v4.4s, v17.4s // (t7, t6) fadd v4.4s, v4.4s, v17.4s // (t4, t5) fsub v27.4s, v2.4s, v20.4s // (t11, t10) fadd v2.4s, v2.4s, v20.4s // (t8, t9) // the 3 direct outputs: out[0,10,5] = dc[0,1,2] + t[0,4,8] + t[1,5,9] ext v16.16b, v22.16b, v22.16b, #8 // (t1, t0) zip1 v17.2d, v4.2d, v2.2d // (t4, t8) zip2 v20.2d, v4.2d, v2.2d // (t5, t9) fadd v16.4s, v16.4s, v22.4s // t[0]+t[1] fadd v17.4s, v17.4s, v20.4s // (t[4]+t[5], t[8]+t[9]) fadd v16.4s, v16.4s, v0.4s // out[0] fadd v17.4s, v17.4s, v1.4s // (out[10], out[5]) st1 { v16.d }[0], [x1] st1 { v17.d }[1], [x8] st1 { v17.d }[0], [x9] // twiddles; swap(t * tab) = swap(t) * tab as every multiplier is // dup-symmetric. lo chunks seed the accumulator with dc[] (= the // output stage's dc preadd): m = dc + t*v25 - swap(t)*v28; hi chunks // r = t*v29 + swap(t)*v10, v10's +- giving r[3] += t[2]/r[2] -= t[3] // in half-swapped order. Accumulates are spread out for the A53 ext v16.16b, v22.16b, v22.16b, #8 // (t1, t0) mov v6.16b, v0.16b // m0 = dc[0] ext v17.16b, v21.16b, v21.16b, #8 // (t2, t3) fmul v7.4s, v21.4s, v29.4s // (r3, r2) fmla v6.4s, v22.4s, v25.4s // m0 += t[0,1]*r_lo dup v18.2d, v1.d[0] // m1 = dc[1] fmla v18.4s, v4.4s, v25.4s // m1 += t[4,5]*r_lo fmls v6.4s, v16.4s, v28.4s // m0 -= swap*nt_lo ext v16.16b, v4.16b, v4.16b, #8 // (t5, t4) fmla v7.4s, v17.4s, v10.4s // (r3, r2) += swap*nt_hi ext v17.16b, v26.16b, v26.16b, #8 // (t6, t7) fmls v18.4s, v16.4s, v28.4s // m1 -= swap*nt_lo fmul v23.4s, v26.4s, v29.4s // (r7, r6) dup v19.2d, v1.d[1] // m2 = dc[2] fmla v19.4s, v2.4s, v25.4s // m2 += t[8,9]*r_lo ext v16.16b, v2.16b, v2.16b, #8 // (t9, t8) fmla v23.4s, v17.4s, v10.4s // (r7, r6) ext v17.16b, v27.16b, v27.16b, #8 // (t10, t11) fmul v5.4s, v27.4s, v29.4s // (r11, r10) fmls v19.4s, v16.4s, v28.4s // m2 -= swap*nt_lo fmla v5.4s, v17.4s, v10.4s // (r11, r10) // output butterflies around rot(x) = (x.im, -x.re): out = m +- rot(r_hi). // The half-swap makes rev(r_hi) a plain rev64, and u = rev*v31 is // exact (+-1.0), so each +-rot pair is one non-destructive fsub/fadd rev64 v16.4s, v7.4s // (r3.im, r3.re, r2.im, r2.re) rev64 v17.4s, v23.4s // (r7.im, r7.re, r6.im, r6.re) rev64 v20.4s, v5.4s // (r11.im, ..., r10.re) fmul v16.4s, v16.4s, v31.4s // u0 fmul v17.4s, v17.4s, v31.4s // u1 fmul v20.4s, v20.4s, v31.4s // u2 fsub v7.4s, v6.4s, v16.4s // (out6, out3) = m0 + rot fadd v6.4s, v6.4s, v16.4s // (out9, out12) = m0 - rot fsub v23.4s, v18.4s, v17.4s // (out1, out13) fadd v22.4s, v18.4s, v17.4s // (out4, out7) fsub v5.4s, v19.4s, v20.4s // (out11, out8) fadd v4.4s, v19.4s, v20.4s // (out14, out2) add x10, x1, x6, lsl #1 // &out[6] add x11, x1, x6 // &out[3] st1 { v7.d }[0], [x10] st1 { v7.d }[1], [x11] add x12, x8, x3, lsl #2 // &out[9] add x13, x1, x6, lsl #2 // &out[12] st1 { v6.d }[0], [x12] st1 { v6.d }[1], [x13] add x10, x1, x3 // &out[1] add x11, x9, x6 // &out[13] st1 { v23.d }[0], [x10] st1 { v23.d }[1], [x11] add x12, x1, x3, lsl #2 // &out[4] add x13, x8, x3, lsl #1 // &out[7] st1 { v22.d }[0], [x12] st1 { v22.d }[1], [x13] add x10, x9, x3 // &out[11] add x11, x1, x3, lsl #3 // &out[8] st1 { v5.d }[0], [x10] st1 { v5.d }[1], [x11] add x12, x9, x3, lsl #2 // &out[14] add x13, x1, x3, lsl #1 // &out[2] st1 { v4.d }[0], [x12] st1 { v4.d }[1], [x13] .endm // LIBOPUS DELTA (I): the standalone ff_tx_fft15_(ns_)float_neon functions are // not instantiated; the PFA kernel below inlines FFT15_LOAD/FFT15_CORE. // 15xM PFA (len = 15*M, M a power of two), like the x86 fft_pfa_15xM: // dim1 = M 15pt transforms scattered into s->tmp at sub_map[i], spaced M // apart; dim2 = 15 in-place M-pt subtransforms (plain blr, the uniform ABI // needs no asm-call variant); post = out[i*stride] = s->tmp[out_map[i]]. // AVTXContext offsets: len=0, map=8, tmp=24, sub=32, fn[0]=40. .macro PFA_15_FN name, no_perm function celt_tx_fft_pfa_15xM_\name\()_neon, export=1 // LIBOPUS DELTA (P) AARCH64_SIGN_LINK_REGISTER // LIBOPUS DELTA (F) stp x29, x30, [sp, #-128]! mov x29, sp // LIBOPUS DELTA (F) stp x19, x20, [sp, #16] stp x21, x22, [sp, #32] stp x23, x24, [sp, #48] stp x25, x26, [sp, #64] stp x27, x28, [sp, #80] stp d8, d9, [sp, #96] str d10, [sp, #112] mov x25, x0 // root context mov x26, x1 // user out mov x27, x3 // user stride (bytes) ldr w19, [x0, #0] // len = 15*M ldr x20, [x0, #24] // s->tmp ldr x23, [x0, #32] // s->sub ldr w24, [x23, #0] // M ldr x28, [x23, #8] // sub_map lsl x3, x24, #3 // M*8 = 15pt output stride .if \no_perm == 0 ldr x4, [x0, #8] // in_map, advanced by the loads .endif movrel x5, X(celt_tx_tab_53) ld1 { v28.4s, v29.4s, v30.4s }, [x5] movrel x5, tab_15pt ld1 { v24.4s }, [x5] LOAD_SUBADD // v31 FFT15_DERIVE_CONSTS add x6, x3, x3, lsl #1 // stride*3 add x7, x3, x3, lsl #2 // stride*5 1: ldrh w10, [x28], #2 // sub_map[i] (int16) add x1, x20, x10, lsl #3 FFT15_LOAD \no_perm, advance=1 FFT15_CORE hoist_strides=1 subs w19, w19, #15 b.gt 1b ldr x28, [x25, #40] // ctx->fn[0] mov x21, x20 // column base mov w19, #15 2: mov x0, x23 mov x1, x21 mov x2, x21 mov x3, #8 blr x28 // M-point FFT, in-place add x21, x21, x24, lsl #3 subs w19, w19, #1 b.gt 2b // out[i*stride] = s->tmp[out_map[i]], unrolled by 4 (len % 60 == 0) ldr w19, [x25, #0] // len ldr x22, [x25, #8] // s->map (out scatter map only) mov x10, x26 lsl x11, x27, #1 // 2*stride add x12, x27, x11 // 3*stride 3: ldp w13, w15, [x22], #8 // out_map[i..i+3], int16 packed ubfx w14, w13, #16, #16 // out_map[i+1] uxth w13, w13 // out_map[i] ubfx w16, w15, #16, #16 // out_map[i+3] uxth w15, w15 // out_map[i+2] ldr d0, [x20, x13, lsl #3] ldr d1, [x20, x14, lsl #3] ldr d2, [x20, x15, lsl #3] ldr d3, [x20, x16, lsl #3] str d0, [x10] str d1, [x10, x27] str d2, [x10, x11] str d3, [x10, x12] add x10, x10, x11, lsl #1 subs w19, w19, #4 b.gt 3b ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x23, x24, [sp, #48] ldp x25, x26, [sp, #64] ldp x27, x28, [sp, #80] ldp d8, d9, [sp, #96] ldr d10, [sp, #112] ldp x29, x30, [sp], #128 AARCH64_VALIDATE_LINK_REGISTER // LIBOPUS DELTA (F) ret endfunc celt_tx_fft_pfa_15xM_\name\()_neon .endm PFA_15_FN ns_float, 1 // LIBOPUS DELTA (I): PFA_15_FN float (runtime-LUT variant) not instantiated. // NOTE: if reinstated, its FFT15_LOAD no_perm==0 in_map gather still reads the // map as 32-bit words; the maps are now opus_int16, so those loads need the // same halfword/extract treatment applied to the ns/out_map/mdct paths. The // in half of the PFA map is also no longer baked at all (it was unread here // and equal to the mdct gather map >> 1), so the data itself would have to // come back too. // Inverse MDCT (see ff_tx_mdct_inv): pre-rotation, in-place N/2pt FFT, // post-rotation. out is the N/2-complex buffer z, in the N-real input. // Where ld2 can split the re/im planes the multiplies are planar and fused // (fmul + fmls/fmla); the gather and the odd tail use interleaved pairs // (trn1/trn2 broadcast, rev64 swap, the v31 fmla). // LIBOPUS DELTA (T): exp holds only the natural-order twiddles. FFmpeg also // keeps a second copy pre-permuted by the gather map in exp[0..len2) for the // strided path; the values are identical, so the gather loop below indexes // the one table through the map it already loads. // AVTXContext offsets: len=0, map=8 (doubled in init), exp=16, sub=32, fn=40. function celt_tx_mdct_inv_float_neon, export=1 // LIBOPUS DELTA (P) AARCH64_SIGN_LINK_REGISTER // LIBOPUS DELTA (F) stp x29, x30, [sp, #-48]! mov x29, sp // LIBOPUS DELTA (F) stp x19, x20, [sp, #16] stp x21, x22, [sp, #32] mov x19, x1 // z mov x20, x0 // ctx ldr w21, [x0, #0] // N ldr x22, [x0, #16] // exp ldr x4, [x0, #8] // map (doubled) LOAD_SUBADD sub x5, x21, #1 madd x14, x5, x3, x2 // in2 = in + (N-1)*stride lsr w7, w21, #1 // len2 cmp x3, #4 // contiguous input and b.ne 6f // len2 % 4 == 0 takes the tst w21, #7 // single-sweep path b.eq 3f 6: // pre-rotation via gather: z[i] = (in2[-k], in1[k])*exp[k/2], 2 // cx/iter (the map is pre-doubled, so k*4 = (k/2)*sizeof(complex)) mov x13, x19 1: ldr w5, [x4], #4 // k0, k1 (int16 packed) lsr w6, w5, #16 // k1 uxth w5, w5 // k0 madd x8, x5, x3, x2 // &in[k0] (im0) msub x9, x5, x3, x14 // &in2[-k0] (re0) madd x10, x6, x3, x2 // &in[k1] (im1) msub x11, x6, x3, x14 // &in2[-k1] (re1) add x15, x22, x5, lsl #2 // &exp[k0/2] add x16, x22, x6, lsl #2 // &exp[k1/2] ldr s0, [x9] ld1 { v0.s }[1], [x8] ld1 { v0.s }[2], [x11] ld1 { v0.s }[3], [x10] // (re0, im0, re1, im1) = tmp ldr d1, [x15] ld1 { v1.d }[1], [x16] // exp[k0/2], exp[k1/2] trn1 v2.4s, v0.4s, v0.4s // re dup trn2 v3.4s, v0.4s, v0.4s // im dup rev64 v4.4s, v1.4s // exp swapped fmul v5.4s, v2.4s, v1.4s fmul v6.4s, v3.4s, v4.4s fmla v5.4s, v6.4s, v31.4s // z = tmp*exp st1 { v5.4s }, [x13], #16 subs w7, w7, #2 b.gt 1b b 4f // pre-rotation, contiguous: the input is effectively (im, re, im, ...) // interleaved, so sweep it once from both ends with the planes // ld2-split: low ims pair with high res (outputs p, p+1) and high ims // with low res (outputs p'-1, p', p' = len2-1-p), scattered through // the inverse map (2nd half of s->map) with the natural twiddles 3: mov x15, x22 // exp (asc) add x4, x4, x21 // map + len2 (int16) (asc) sub x10, x14, #12 // &in[N-4] (desc) add x12, x22, x21, lsl #2 sub x12, x12, #16 // exp + (len2-2) (desc) add x13, x4, x21 // map + N (int16) sub x13, x13, #4 // map + (N-2) (desc) 5: ld2 { v6.2s, v7.2s }, [x2], #16 // (im_p, im_p1) (re_p', re_p'm1) ld2 { v16.2s, v17.2s }, [x10] // (im_p'm1, im_p') (re_p1, re_p) sub x10, x10, #16 rev64 v17.2s, v17.2s // (re_p, re_p1) ld2 { v1.2s, v2.2s }, [x15], #16 // A: er, ei rev64 v7.2s, v7.2s // (re_p'm1, re_p') ld2 { v3.2s, v4.2s }, [x12] // B: er, ei sub x12, x12, #16 fmul v5.2s, v17.2s, v1.2s // A: re*er fmul v18.2s, v17.2s, v2.2s // A: re*ei fmul v19.2s, v7.2s, v3.2s // B: re*er fmul v20.2s, v7.2s, v4.2s // B: re*ei fmls v5.2s, v6.2s, v2.2s // A: z.re plane fmla v18.2s, v6.2s, v1.2s // A: z.im plane fmls v19.2s, v16.2s, v4.2s // B: z.re plane fmla v20.2s, v16.2s, v3.2s // B: z.im plane ldr w16, [x4], #4 // inv_map[p, p+1] (int16 packed) lsr w17, w16, #16 // inv_map[p+1] uxth w16, w16 // inv_map[p] zip1 v0.2s, v5.2s, v18.2s // z_p zip2 v1.2s, v5.2s, v18.2s // z_p1 str d0, [x19, w16, uxtw #3] str d1, [x19, w17, uxtw #3] ldr w16, [x13] // inv_map[p'-1, p'] (int16 packed) sub x13, x13, #4 lsr w17, w16, #16 // inv_map[p'] uxth w16, w16 // inv_map[p'-1] zip1 v2.2s, v19.2s, v20.2s // z_p'm1 zip2 v3.2s, v19.2s, v20.2s // z_p' str d2, [x19, w16, uxtw #3] str d3, [x19, w17, uxtw #3] subs w7, w7, #4 b.gt 5b 4: ldr x5, [x20, #40] // fn[0] ldr x0, [x20, #32] // sub[0] mov x1, x19 mov x2, x19 mov x3, #8 blr x5 // N/2pt FFT, in-place // post-rotation over symmetric pairs (i0 = len4+i, i1 = len4-1-i), // 2 pairs/iter, planes ld2-split: r = swap(z)*swap(e), the im parts // crossing partners (z[i0] = (r_i0.re, r_i1.im) and vice versa), so // each side stores its re plane zipped with the other's reversed ims mov x15, x22 // exp_post = exp (natural order) add x8, x19, x21, lsl #1 // p_i0 = z + len4 sub x9, x8, #16 // p_i1 = z + len4 - 2 add x10, x15, x21, lsl #1 // e_i0 = exp_post + len4 sub x11, x10, #16 // e_i1 = exp_post + len4 - 2 lsr w7, w21, #2 // len4 lsr w6, w7, #1 // pairs of pairs cbz w6, 8f // N == 4: lone middle pair 2: ld2 { v0.2s, v1.2s }, [x8] // z asc (i0, i0+1): re, im planes ld2 { v2.2s, v3.2s }, [x9] // z desc (i1-1, i1) ld2 { v4.2s, v5.2s }, [x10], #16 // e asc: er, ei ld2 { v6.2s, v7.2s }, [x11] // e desc sub x11, x11, #16 fmul v16.2s, v1.2s, v5.2s // asc: z.im*e.im fmul v18.2s, v1.2s, v4.2s // asc: z.im*e.re fmul v20.2s, v3.2s, v7.2s // desc: z.im*e.im fmul v22.2s, v3.2s, v6.2s // desc: z.im*e.re fmls v16.2s, v0.2s, v4.2s // re plane (r_i0, r_i0+1) fmla v18.2s, v0.2s, v5.2s // im plane (r_i0, r_i0+1) fmls v20.2s, v2.2s, v6.2s // re plane (r_i1-1, r_i1) fmla v22.2s, v2.2s, v7.2s // im plane (r_i1-1, r_i1) rev64 v22.2s, v22.2s // (r_i1.im, r_i1-1.im) rev64 v18.2s, v18.2s // (r_i0+1.im, r_i0.im) zip1 v0.4s, v16.4s, v22.4s // (z[i0], z[i0+1]) zip1 v2.4s, v20.4s, v18.4s // (z[i1-1], z[i1]) st1 { v0.4s }, [x8], #16 st1 { v2.4s }, [x9] sub x9, x9, #16 subs w6, w6, #1 b.gt 2b 8: // odd len4 (N % 8 == 4): one leftover pair, (i0, i1) = (len2-1, 0). // x8/x10 already point at it; x9/x11 sit one complex below their slot. tbz w7, #0, 9f LOAD_SUBADD // v31 (clobbered by the FFT) add x9, x9, #8 add x11, x11, #8 ldr d0, [x8] // z[i0] ld1 { v0.d }[1], [x9] // v0 = (z[i0], z[i1]) ldr d1, [x10] // exp[i0] ld1 { v1.d }[1], [x11] // v1 = (exp[i0], exp[i1]) rev64 v2.4s, v0.4s // swap(z) = a rev64 v3.4s, v1.4s // swap(exp) = b trn1 v4.4s, v2.4s, v2.4s // a.re dup trn2 v5.4s, v2.4s, v2.4s // a.im dup fmul v6.4s, v4.4s, v3.4s // a.re*b fmul v7.4s, v5.4s, v1.4s // a.im*b_swap (b_swap = orig exp) fmla v6.4s, v7.4s, v31.4s // v6 = (r0.re, r0.im, r1.re, r1.im) mov v16.16b, v6.16b ins v16.s[1], v6.s[3] // (r0.re, r1.im, r1.re, r1.im) ins v16.s[3], v6.s[1] // (r0.re, r1.im, r1.re, r0.im) st1 { v16.d }[0], [x8] // z[i0] = (r0.re, r1.im) st1 { v16.d }[1], [x9] // z[i1] = (r1.re, r0.im) 9: ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x29, x30, [sp], #48 AARCH64_VALIDATE_LINK_REGISTER // LIBOPUS DELTA (F) ret endfunc celt_tx_mdct_inv_float_neon .macro SETUP_SR_RECOMB len, re, im, dec ldr w5, =(\len - 4*7) movrel \re, X(celt_tx_tab_\len\()) add \im, \re, x5 mov \dec, #-32 .if \len > 32 mov x21, #2*\len add x22, x21, x21, lsl #1 .endif .endm .macro SR_COMBINE e0, e1, e2, e3, e4, e5, e6, e7, \ o0, o1, o2, o3, o4, o5, o6, o7, \ re, im, dec, swap_im, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, \ t6=v22, t7=v23, t8=v24, t9=v25, ta=v26, tb=v27 ld1 { \t8\().4s, \t9\().4s }, [\im], \dec ld1 { \t0\().4s, \t1\().4s }, [\re], #32 .if \swap_im == 1 ext \t2\().16b, \t9\().16b, \t9\().16b, #8 ext \t3\().16b, \t8\().16b, \t8\().16b, #8 .else ext \t2\().16b, \t8\().16b, \t8\().16b, #8 ext \t3\().16b, \t9\().16b, \t9\().16b, #8 .endif trn1 \t4\().4s, \t0\().4s, \t0\().4s // cos0022 trn2 \t0\().4s, \t0\().4s, \t0\().4s // cos4466 trn1 \t5\().4s, \t1\().4s, \t1\().4s // cos1133 trn2 \t1\().4s, \t1\().4s, \t1\().4s // cos5577 rev64 \t6\().4s, \o0\().4s // E m2[0,1].imre rev64 \t7\().4s, \o2\().4s // O m2[0,1].imre rev64 \t8\().4s, \o4\().4s // E m2[2,3].imre rev64 \t9\().4s, \o6\().4s // O m2[2,3].imre fmul \t6\().4s, \t6\().4s, \t4\().4s // E m2[0,1].imre*t1[0,2] fmul \t7\().4s, \t7\().4s, \t0\().4s // O m2[0,1].imre*t1[0,2] fmul \t8\().4s, \t8\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t9\().4s, \t9\().4s, \t0\().4s // O m2[2,3].imre*t1[0,2] rev64 \ta\().4s, \o1\().4s // E m3[0,1].imre rev64 \tb\().4s, \o3\().4s // O m3[0,1].imre rev64 \t4\().4s, \o5\().4s // E m3[2,3].imre rev64 \t0\().4s, \o7\().4s // O m3[2,3].imre fmul \ta\().4s, \ta\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmul \tb\().4s, \tb\().4s, \t1\().4s // O m3[0,1].imre*t1[4,6] fmul \t4\().4s, \t4\().4s, \t5\().4s // E m3[2,3].imre*t1[4,6] fmul \t0\().4s, \t0\().4s, \t1\().4s // O m3[2,3].imre*t1[4,6] trn1 \t5\().4s, \t3\().4s, \t3\().4s // wim2200 trn2 \t3\().4s, \t3\().4s, \t3\().4s // wim3311 trn1 \t1\().4s, \t2\().4s, \t2\().4s // wim6644 trn2 \t2\().4s, \t2\().4s, \t2\().4s // wim7755 fmul \t5\().4s, \t5\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmla \t7\().4s, \o2\().4s, \t5\().4s // O w0123 fmls \t9\().4s, \o6\().4s, \t5\().4s // O j0123 fmla \t6\().4s, \o0\().4s, \t3\().4s // E w0123 fmls \t8\().4s, \o4\().4s, \t3\().4s // E j0123 fmla \ta\().4s, \o1\().4s, \t2\().4s // E w4567 fmla \tb\().4s, \o3\().4s, \t1\().4s // O w4567 fmls \t4\().4s, \o5\().4s, \t2\().4s // E j4567 fmls \t0\().4s, \o7\().4s, \t1\().4s // O j4567 fsub \t2\().4s, \t7\().4s, \t9\().4s fsub \t1\().4s, \t8\().4s, \t6\().4s fsub \t3\().4s, \t4\().4s, \ta\().4s fsub \t5\().4s, \t0\().4s, \tb\().4s fadd \t6\().4s, \t8\().4s, \t6\().4s fadd \t7\().4s, \t9\().4s, \t7\().4s fadd \t8\().4s, \t4\().4s, \ta\().4s fadd \t9\().4s, \t0\().4s, \tb\().4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t5\().4s, \t5\().4s, v31.4s rev64 \t6\().4s, \t6\().4s rev64 \t8\().4s, \t8\().4s rev64 \t7\().4s, \t7\().4s rev64 \t9\().4s, \t9\().4s fsub \o0\().4s, \e0\().4s, \t6\().4s fsub \o1\().4s, \e1\().4s, \t8\().4s fsub \o2\().4s, \e2\().4s, \t1\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s fsub \o4\().4s, \e4\().4s, \t7\().4s fsub \o5\().4s, \e6\().4s, \t9\().4s fadd \o6\().4s, \e5\().4s, \t2\().4s fsub \o7\().4s, \e7\().4s, \t5\().4s fadd \e0\().4s, \e0\().4s, \t6\().4s fadd \e1\().4s, \e1\().4s, \t8\().4s fadd \e2\().4s, \e2\().4s, \t1\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s fadd \e4\().4s, \e4\().4s, \t7\().4s fsub \e5\().4s, \e5\().4s, \t2\().4s // swapped fadd \e6\().4s, \e6\().4s, \t9\().4s // swapped fadd \e7\().4s, \e7\().4s, \t5\().4s .endm .macro SR_COMBINE_HALF e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, t3, t4, t5, part .if \part == 0 trn1 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn1 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn2 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .endif .if \part == 0 trn2 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn2 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn1 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t5\().4s, \t5\().4s, v31.4s fmul \c3\().4s, \c3\().4s, v31.4s rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \t3\().4s, \o3\().4s // E m3[2,3].imre fmul \o0\().4s, \o0\().4s, \c3\().4s // E m2[0,1].imre*t1[0,2] fmul \o1\().4s, \o1\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmla \o0\().4s, \t0\().4s, \t4\().4s // E w0123 fmla \o1\().4s, \t2\().4s, \c1\().4s // E w4567 fmul \t1\().4s, \t1\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t3\().4s, \t3\().4s, \c1\().4s // E m3[2,3].imre*t1[4,6] fmls \t1\().4s, \o2\().4s, \c3\().4s // E j0123 fmls \t3\().4s, \o3\().4s, \t5\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \t1\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \t3\().4s, \o1\().4s fsub \t3\().4s, \t3\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s rev64 \t1\().4s, \t1\().4s rev64 \t2\().4s, \t2\().4s .if \part == 0 fsub \o0\().4s, \e0\().4s, \t1\().4s fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s .else fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \o2\().4s, \e1\().4s, \t2\().4s fsub \o1\().4s, \e2\().4s, \t0\().4s fadd \o3\().4s, \e3\().4s, \t3\().4s .endif .if \part == 0 fadd \e0\().4s, \e0\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s .else fadd \e0\().4s, \e0\().4s, \t1\().4s fsub \e1\().4s, \e1\().4s, \t2\().4s // swapped fadd \e2\().4s, \e2\().4s, \t0\().4s // swapped fsub \e3\().4s, \e3\().4s, \t3\().4s .endif .endm /* Same as SR_COMBINE_HALF, but heroically tries to use 3 temporary registers * without touching the tables. */ .macro SR_COMBINE_LITE e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, part rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre .if \part == 0 trn2 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t2\().4s, \t2\().4s, v31.4s fmul \o2\().4s, \o2\().4s, \t2\().4s fmul \o0\().4s, \o0\().4s, \t2\().4s // E m2[0,1].imre*t1[0,2] .if \part == 0 trn1 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .else trn2 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .endif fmul \t1\().4s, \t1\().4s, \t2\().4s // E m2[2,3].imre*t1[0,2] fmla \o0\().4s, \t0\().4s, \t2\().4s // E w0123 fsub \t1\().4s, \t1\().4s, \o2\().4s // E j0123 rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \o2\().4s, \o3\().4s // E m3[2,3].imre .if \part == 0 trn2 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .else trn1 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .endif fmul \t0\().4s, \t0\().4s, v31.4s fmul \o1\().4s, \o1\().4s, \t0\().4s // E m3[0,1].imre*t1[4,6] fmul \o3\().4s, \o3\().4s, \t0\().4s .if \part == 0 trn1 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .endif fmul \o2\().4s, \o2\().4s, \t0\().4s // E m3[2,3].imre*t1[4,6] fmla \o1\().4s, \t2\().4s, \t0\().4s // E w4567 fsub \o2\().4s, \o2\().4s, \o3\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \o0\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \o2\().4s, \o1\().4s fsub \t1\().4s, \o2\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s rev64 \t2\().4s, \t2\().4s rev64 \o0\().4s, \o0\().4s .if \part == 0 fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t1\().4s .else fadd \o2\().4s, \e1\().4s, \t0\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o3\().4s, \e3\().4s, \t1\().4s .endif .if \part == 0 fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t1\().4s .else fsub \e1\().4s, \e1\().4s, \t0\().4s // swapped fadd \e2\().4s, \e2\().4s, \t2\().4s // swapped fsub \e3\().4s, \e3\().4s, \t1\().4s .endif mov \t1\().16b, \o0\().16b fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \e0\().4s, \e0\().4s, \t1\().4s .endm .macro SR_COMBINE_4 len, part, off add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 ldp q0, q1, [x1, #((0 + \part)*32 + \off)] ldp q4, q5, [x1, #((2 + \part)*32 + \off)] ldp q2, q3, [x10, #((0 + \part)*32 + \off)] ldp q6, q7, [x10, #((2 + \part)*32 + \off)] ldp q8, q9, [x11, #((0 + \part)*32 + \off)] ldp q10, q11, [x11, #((2 + \part)*32 + \off)] ldp q12, q13, [x12, #((0 + \part)*32 + \off)] ldp q14, q15, [x12, #((2 + \part)*32 + \off)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q0, q1, [x1, #((0 + \part)*32 + \off)] stp q4, q5, [x1, #((2 + \part)*32 + \off)] stp q2, q3, [x10, #((0 + \part)*32 + \off)] stp q6, q7, [x10, #((2 + \part)*32 + \off)] stp q8, q9, [x11, #((0 + \part)*32 + \off)] stp q12, q13, [x11, #((2 + \part)*32 + \off)] stp q10, q11, [x12, #((0 + \part)*32 + \off)] stp q14, q15, [x12, #((2 + \part)*32 + \off)] .endm .macro SR_COMBINE_FULL len, off=0 add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 SR_COMBINE_4 \len, 0, \off SR_COMBINE_4 \len, 1, \off SR_COMBINE_4 \len, 4, \off SR_COMBINE_4 \len, 5, \off .endm .macro SR_COMBINE_D2 part, off add x10, x1, #((\part)*32 + \off) add x11, x14, #((\part)*32 + \off) add x12, x15, #((\part)*32 + \off) add x13, x16, #((\part)*32 + \off) ldp q0, q1, [x10] ldp q4, q5, [x10, #(2*32)] ldp q2, q3, [x11] ldp q6, q7, [x11, #(2*32)] ldp q8, q9, [x12] ldp q10, q11, [x12, #(2*32)] ldp q12, q13, [x13] ldp q14, q15, [x13, #(2*32)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d zip1 v20.2d, v2.2d, v6.2d zip2 v21.2d, v2.2d, v6.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d ldp q0, q1, [x10, #(1*32)] ldp q4, q5, [x10, #(3*32)] ldp q2, q3, [x11, #(1*32)] ldp q6, q7, [x11, #(3*32)] st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10], #64 st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x11], #64 zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d ldp q8, q9, [x12, #(1*32)] ldp q10, q11, [x12, #(3*32)] ldp q12, q13, [x13, #(1*32)] ldp q14, q15, [x13, #(3*32)] st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12], #64 st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13], #64 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10] zip1 v16.2d, v2.2d, v6.2d zip2 v17.2d, v2.2d, v6.2d zip1 v18.2d, v3.2d, v7.2d zip2 v19.2d, v3.2d, v7.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x11] zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12] zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13] .endm .macro SR_COMBINE_DINT off=0 add x14, x1, x21 add x15, x1, x21, lsl #1 add x16, x1, x22 SR_COMBINE_D2 0, \off SR_COMBINE_D2 4, \off .endm .macro FFT32_FN name, no_perm function celt_tx_fft32_\name\()_neon, export=1 // LIBOPUS DELTA (P) stp d14, d15, [sp, #-16*4]! stp d8, d9, [sp, #16*3] stp d10, d11, [sp, #16*2] stp d12, d13, [sp, #16] LOAD_SUBADD SETUP_SR_RECOMB 32, x7, x8, x9 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v5, v6, v7 SR_COMBINE v0, v1, v2, v3, v4, v5, v6, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v6.2d zip2 v19.2d, v1.2d, v6.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1], #64 zip1 v20.2d, v2.2d, v5.2d zip2 v21.2d, v2.2d, v5.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v8.2d, v12.2d zip2 v25.2d, v8.2d, v12.2d zip1 v26.2d, v9.2d, v13.2d zip2 v27.2d, v9.2d, v13.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1], #64 zip1 v28.2d, v10.2d, v14.2d zip2 v29.2d, v10.2d, v14.2d zip1 v30.2d, v11.2d, v15.2d zip2 v31.2d, v11.2d, v15.2d st1 { v28.4s, v29.4s, v30.4s, v31.4s }, [x1] ldp d12, d13, [sp, #16] ldp d10, d11, [sp, #16*2] ldp d8, d9, [sp, #16*3] ldp d14, d15, [sp], #16*4 ret endfunc celt_tx_fft32_\name\()_neon .endm FFT32_FN ns_float, 1 // LIBOPUS DELTA (I): FFT32_FN float not instantiated. // LIBOPUS DELTA (S): cmp_imm and SR_TRANSFORM_DEF (the len >= 2048 transform // chain) are trimmed; CELT's largest split-radix sub-FFT is 512 (power-of-two // custom modes), so the 1024 block and above are removed below. .macro FFT_SPLIT_RADIX_FN name, no_perm function celt_tx_fft_sr_\name\()_neon, export=1 // LIBOPUS DELTA (P) stp x21, x22, [sp, #-16*6]! stp d8, d9, [sp, #16*5] stp d10, d11, [sp, #16*4] stp d12, d13, [sp, #16*3] stp d14, d15, [sp, #16*2] stp x19, x20, [sp, #16] ldr w19, [x0, #0] // global target mov w20, w19 // local length LOAD_SUBADD SETUP_LUT \no_perm 32: SETUP_SR_RECOMB 32, x7, x8, x9 LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 6, 5, 7, x2, \no_perm, 1 LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v6, v5, v7 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q2, q3, [x1, #32*1] stp q6, q7, [x1, #32*3] stp q10, q11, [x1, #32*5] stp q14, q15, [x1, #32*7] cmp w20, #32 b.gt 64f stp q0, q1, [x1, #32*0] stp q4, q5, [x1, #32*2] stp q8, q9, [x1, #32*4] stp q12, q13, [x1, #32*6] ret 64: SETUP_SR_RECOMB 64, x7, x8, x9 LOAD_INPUT 2, 3, 10, 11, x2, \no_perm, 1 LOAD_INPUT 6, 14, 7, 15, x2, \no_perm, 1 FFT16 v2, v3, v10, v11, v6, v14, v7, v15 LOAD_INPUT 16, 17, 18, 19, x2, \no_perm LOAD_INPUT 20, 22, 21, 23, x2, \no_perm, 1 FFT16 v16, v17, v18, v19, v20, v22, v21, v23, \ v24, v25, v26, v27, v28, v29, v30 ld1 { v26.4s, v27.4s }, [x8], x9 ldp q24, q25, [x7], #32 ext v26.16b, v26.16b, v26.16b, #8 ext v27.16b, v27.16b, v27.16b, #8 cmp w19, #64 b.eq 2f // custom deinterleave // TODO: investigate doing the 2 combines like in deinterleave // TODO: experiment with spilling to gprs and converting to HALF or full SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 stp q0, q1, [x1, #32* 0] stp q8, q9, [x1, #32* 4] stp q2, q3, [x1, #32* 8] stp q16, q17, [x1, #32*12] SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v0, v1, v8, 1 stp q4, q20, [x1, #32* 2] stp q12, q21, [x1, #32* 6] stp q6, q5, [x1, #32*10] stp q7, q13, [x1, #32*14] ldp q2, q3, [x1, #32*1] ldp q6, q7, [x1, #32*3] ldp q12, q13, [x1, #32*5] ldp q16, q17, [x1, #32*7] SR_COMBINE v2, v3, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v8, v0, v1, v4, v5 stp q2, q3, [x1, #32* 1] stp q6, q7, [x1, #32* 3] stp q12, q13, [x1, #32* 5] stp q16, q17, [x1, #32* 7] stp q10, q11, [x1, #32* 9] stp q18, q19, [x1, #32*11] stp q14, q15, [x1, #32*13] stp q22, q23, [x1, #32*15] #ifdef CUSTOM_MODES cmp w20, #64 b.gt 128f #endif ret #ifdef CUSTOM_MODES 128: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #32 add x1, x1, #16*32 bl 32b add x1, x1, #8*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #24*32 SETUP_SR_RECOMB 128, x7, x8, x9 cmp w19, #128 b.eq 0f SR_COMBINE_FULL 128 cmp w20, #128 b.gt 256f ret 256: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #64 add x1, x1, #32*32 bl 32b add x1, x1, #16*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #48*32 SETUP_SR_RECOMB 256, x7, x8, x9 cmp w19, #256 b.eq 0f SR_COMBINE_FULL 256 SR_COMBINE_FULL 256, 8*32 cmp w20, #256 b.gt 512f ret 512: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #128 add x1, x1, #64*32 bl 32b add x1, x1, #32*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #96*32 SETUP_SR_RECOMB 512, x7, x8, x9 // LIBOPUS DELTA (S): lengths > 512 trimmed, so w19 == 512 here and // FFmpeg's `cmp w19, #512; b.eq 0f`, its 4x SR_COMBINE_FULL 512 // loop for the 1024+ parents and the recursive ret are dropped. 0: // general deinterleave loop SR_COMBINE_DINT add x1, x1, #32*8 subs w19, w19, #32*4 b.gt 0b ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret #endif 2: // special case for 64 point deinterleave mov x10, v23.d[0] mov x11, v23.d[1] SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v23, v24, v26, 1 zip1 v23.2d, v0.2d, v4.2d zip2 v24.2d, v0.2d, v4.2d zip1 v25.2d, v1.2d, v20.2d zip2 v26.2d, v1.2d, v20.2d zip1 v27.2d, v8.2d, v12.2d zip2 v28.2d, v8.2d, v12.2d zip1 v29.2d, v9.2d, v21.2d zip2 v30.2d, v9.2d, v21.2d mov v20.16b, v5.16b mov v21.16b, v7.16b mov x12, x1 add x13, x1, #32* 4 add x14, x1, #32* 8 add x15, x1, #32*12 zip1 v4.2d, v2.2d, v6.2d zip2 v5.2d, v2.2d, v6.2d zip1 v6.2d, v3.2d, v20.2d zip2 v7.2d, v3.2d, v20.2d zip1 v0.2d, v16.2d, v21.2d zip2 v1.2d, v16.2d, v21.2d zip1 v2.2d, v17.2d, v13.2d zip2 v3.2d, v17.2d, v13.2d // stp is faster by a little on A53, but this is faster on M1s (theory) ldp q8, q9, [x1, #32*1] ldp q12, q13, [x1, #32*5] st1 { v23.4s, v24.4s, v25.4s, v26.4s }, [x12], #64 // 32* 0...1 st1 { v27.4s, v28.4s, v29.4s, v30.4s }, [x13], #64 // 32* 4...5 st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x14], #64 // 32* 8...9 st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x15], #64 // 32*12..13 mov v23.d[0], x10 mov v23.d[1], x11 ldp q6, q7, [x1, #32*3] ldp q16, q17, [x1, #32*7] SR_COMBINE v8, v9, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v4, v0, v1, v5, v20 zip1 v0.2d, v8.2d, v6.2d zip2 v1.2d, v8.2d, v6.2d zip1 v2.2d, v9.2d, v7.2d zip2 v3.2d, v9.2d, v7.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x12] zip1 v4.2d, v12.2d, v16.2d zip2 v5.2d, v12.2d, v16.2d zip1 v6.2d, v13.2d, v17.2d zip2 v7.2d, v13.2d, v17.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x13] zip1 v0.2d, v10.2d, v18.2d zip2 v1.2d, v10.2d, v18.2d zip1 v2.2d, v11.2d, v19.2d zip2 v3.2d, v11.2d, v19.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x14] zip1 v4.2d, v14.2d, v22.2d zip2 v5.2d, v14.2d, v22.2d zip1 v6.2d, v15.2d, v23.2d zip2 v7.2d, v15.2d, v23.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x15] ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret endfunc celt_tx_fft_sr_\name\()_neon .endm FFT_SPLIT_RADIX_FN ns_float, 1 // LIBOPUS DELTA (I): FFT_SPLIT_RADIX_FN float not instantiated. #endif /* !FIXED_POINT */ #endif /* __aarch64__ */