/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef MOZILLA_GFX_SWIZZLE_NEON_H_
#define MOZILLA_GFX_SWIZZLE_NEON_H_

#ifdef MOZILLA_GFX_SWIZZLE_GENERIC_H_
#  error "SwizzleNEON.h must be included before SwizzleGeneric.h"
#endif

#include <arm_neon.h>

#include "SwizzleGenericDecls.h"

namespace mozilla::gfx {

#ifdef __aarch64__
using mozneon = xsimd::neon64;
#else
using mozneon = xsimd::neon;
#endif

extern const uint32_t sUnpremultiplyTable_NEON[256];

template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE xsimd::batch<uint8_t, Arch> LoadRemainder_SIMD(
    const uint8_t* aSrc, size_t aLength) {
  const uint32_t* src32 = reinterpret_cast<const uint32_t*>(aSrc);
  uint32x4_t dst32;
  if (aLength >= 2) {
    // Load first 2 pixels
    dst32 = vcombine_u32(vld1_u32(src32), vdup_n_u32(0));
    // Load third pixel
    if (aLength >= 3) {
      dst32 = vld1q_lane_u32(src32 + 2, dst32, 2);
    }
  } else {
    // Load single pixel
    dst32 = vld1q_lane_u32(src32, vdupq_n_u32(0), 0);
  }
  return vreinterpretq_u16_u32(dst32);
}

template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE void StoreRemainder_SIMD(
    uint8_t* aDst, size_t aLength, const xsimd::batch<uint8_t, Arch>& aSrc) {
  uint32_t* dst32 = reinterpret_cast<uint32_t*>(aDst);
  uint32x4_t src32 = vreinterpretq_u32_u16(aSrc);
  if (aLength >= 2) {
    // Store first 2 pixels
    vst1_u32(dst32, vget_low_u32(src32));
    // Store third pixel
    if (aLength >= 3) {
      vst1q_lane_u32(dst32 + 2, src32, 2);
    }
  } else {
    // Store single pixel
    vst1q_lane_u32(dst32, src32, 0);
  }
}

// This matches the original specialization, which is cheaper than what xsimd
// and clang generate.
template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE xsimd::batch<uint16_t, Arch> ExtractAlpha_SIMD(
    const xsimd::batch<uint8_t, Arch>& aSrc,
    const xsimd::batch<uint16_t, Arch>& aGreenAlpha) {
  return vtrnq_u16(aGreenAlpha, aGreenAlpha).val[1];
}

// arm devices perform better with the original specialization as aarch64,
// because there is no xsimd::swizzle vectorization.
template <class Arch, bool aSwapRB, bool aOpaqueAlpha>
  requires std::same_as<Arch, xsimd::neon>
static MOZ_ALWAYS_INLINE xsimd::batch<uint8_t, Arch> SwizzleVector_SIMD(
    const xsimd::batch<uint8_t, Arch>& aSrc) {
  if constexpr (aSwapRB) {
    // Swap R and B, then add to G and A (forced to 255):
    // (((src>>16) | (src << 16)) & 0x00FF00FF) |
    //   ((src | 0xFF000000) & ~0x00FF00FF)
    return vbslq_u16(
        vdupq_n_u16(0x00FF), vrev32q_u16(aSrc),
        aOpaqueAlpha
            ? vorrq_u16(aSrc, vreinterpretq_u16_u32(vdupq_n_u32(0xFF000000)))
            : aSrc.data);
  }
  if constexpr (aOpaqueAlpha) {
    return vorrq_u16(aSrc, vreinterpretq_u16_u32(vdupq_n_u32(0xFF000000)));
  }
  return aSrc;
}

template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE xsimd::batch<uint32_t, Arch> UnpremultiplyLookup_SIMD(
    const xsimd::batch<uint8_t, Arch>& aSrc,
    const xsimd::batch<uint16_t, Arch>& aGa) {
  // Extract the alphas for the 4 pixels from the now isolated words.
  alignas(Arch::alignment())
      uint16_t alphaBuf[xsimd::batch<uint16_t, Arch>::size];
  aGa.store_aligned(alphaBuf);

  // Load all of the interleaved low and high portions of the reciprocals
  // and combine them a single vector as lo1 hi1 lo2 hi2 lo3 hi3 lo4 hi4
  return vreinterpretq_u16_u32(vld1q_lane_u32(
      &sUnpremultiplyTable_NEON[alphaBuf[7]],
      vld1q_lane_u32(
          &sUnpremultiplyTable_NEON[alphaBuf[5]],
          vld1q_lane_u32(&sUnpremultiplyTable_NEON[alphaBuf[3]],
                         vld1q_lane_u32(&sUnpremultiplyTable_NEON[alphaBuf[1]],
                                        vdupq_n_u32(0), 0),
                         1),
          2),
      3));
}

template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE xsimd::batch<uint8_t, Arch> UnpremultiplyReverse_SIMD(
    const xsimd::batch<uint8_t, Arch>& aSrc,
    const xsimd::batch<uint32_t, Arch>& aRecip,
    const xsimd::batch<uint16_t, Arch>& aRb,
    const xsimd::batch<uint16_t, Arch>& aGa) {
  // Transpose the interleaved low/high portions so that we produce
  // two separate duplicated vectors for the low and high portions respectively:
  // lo1 lo1 lo2 lo2 lo3 lo3 lo4 lo4 and hi1 hi1 hi2 hi2 hi3 hi3 hi4 hi4
  uint16x8x2_t q1234lohi = vtrnq_u16(aRecip, aRecip);

  // VQDMULH is a signed multiply that doubles (*2) the result, then takes the
  // high word.  To work around the signedness and the doubling, the low
  // portion of the reciprocal only stores the lower 15 bits, which fits in a
  // signed 16 bit integer. The high 9 bit portion is effectively also doubled
  // by 2 as a side-effect of being shifted for storage. Thus the output scale
  // of doing a normal multiply by the high portion and the VQDMULH by the low
  // portion are both doubled and can be safely added together. The resulting
  // sum just needs to be halved (via VHADD) to thus cancel out the doubling.
  // All this combines to produce a reciprocal multiply of the form:
  // rb = ((rb * hi) + ((rb * lo * 2) >> 16)) / 2
  auto rb = vhaddq_u16(vmulq_u16(aRb, q1234lohi.val[1]),
                       vreinterpretq_u16_s16(vqdmulhq_s16(
                           vreinterpretq_s16_u16(aRb),
                           vreinterpretq_s16_u16(q1234lohi.val[0]))));

  // ga = ((ga * hi) + ((ga * lo * 2) >> 16)) / 2
  auto ga = vhaddq_u16(vmulq_u16(aGa, q1234lohi.val[1]),
                       vreinterpretq_u16_s16(vqdmulhq_s16(
                           vreinterpretq_s16_u16(aGa),
                           vreinterpretq_s16_u16(q1234lohi.val[0]))));

  // Combine to the final pixel with ((rb | (ga << 8)) & ~0xFF000000) | (aSrc &
  // 0xFF000000), which inserts back in the original alpha value unchanged.
  return vbslq_u16(vreinterpretq_u16_u32(vdupq_n_u32(0xFF000000)), aSrc,
                   vsliq_n_u16(rb, ga, 8));
}

template <class Arch>
  requires std::same_as<Arch, xsimd::neon> || std::same_as<Arch, xsimd::neon64>
static MOZ_ALWAYS_INLINE xsimd::batch<uint8_t, Arch> LoadRemainderRGB_SIMD(
    const uint8_t* aSrc, size_t aLength) {
  // Load aLength (1-4) packed RGB pixels (3-12 bytes) without reading past the
  // 3*aLength valid source bytes. Bytes beyond the loaded pixels are unused by
  // the expand swizzle and dropped by StoreRemainder_SIMD.
  uint32x4_t px = vdupq_n_u32(0);
  if (aLength >= 2) {
    // Bytes 0-3: pixel 0 (RGB) and pixel 1's R.
    px = vld1q_lane_u32(reinterpret_cast<const uint32_t*>(aSrc), px, 0);
    if (aLength >= 3) {
      // Bytes 4-7: pixel 1's GB and pixel 2's RG.
      px = vld1q_lane_u32(reinterpret_cast<const uint32_t*>(aSrc + 4), px, 1);
      if (aLength >= 4) {
        // Bytes 8-11: pixel 2's B and pixel 3 (RGB).
        px = vld1q_lane_u32(reinterpret_cast<const uint32_t*>(aSrc + 8), px, 2);
      } else {
        // 3 pixels = 9 bytes; byte 8 is pixel 2's B.
        return vld1q_lane_u8(aSrc + 8, vreinterpretq_u8_u32(px), 8);
      }
    } else {
      // 2 pixels = 6 bytes; bytes 4-5 are pixel 1's GB.
      return vreinterpretq_u8_u16(
          vld1q_lane_u16(reinterpret_cast<const uint16_t*>(aSrc + 4),
                         vreinterpretq_u16_u32(px), 2));
    }
    return vreinterpretq_u8_u32(px);
  }
  // 1 pixel = 3 bytes: bytes 0-1 then byte 2.
  uint8x16_t p8 = vreinterpretq_u8_u16(vld1q_lane_u16(
      reinterpret_cast<const uint16_t*>(aSrc), vreinterpretq_u16_u32(px), 0));
  return vld1q_lane_u8(aSrc + 2, p8, 2);
}

// arm devices perform better with the original specialization as aarch64,
// because there is no xsimd::swizzle vectorization.
template <class Arch, bool aSwapRB>
  requires std::same_as<Arch, xsimd::neon>
static MOZ_ALWAYS_INLINE xsimd::batch<uint8_t, Arch> UnpackBatchRGB24_SIMD(
    const xsimd::batch<uint8_t, Arch>& aSrc) {
  uint8x16_t alpha = vreinterpretq_u8_u32(vdupq_n_u32(0xFF000000));
  uint8x8_t masklo;
  uint8x8_t maskhi;
  if constexpr (aSwapRB) {
    static const uint8_t masklo_data[] = {2, 1, 0, 0, 5, 4, 3, 0};
    static const uint8_t maskhi_data[] = {4, 3, 2, 0, 7, 6, 5, 0};
    masklo = vld1_u8(masklo_data);
    maskhi = vld1_u8(maskhi_data);
  } else {
    static const uint8_t masklo_data[] = {0, 1, 2, 0, 3, 4, 5, 0};
    static const uint8_t maskhi_data[] = {2, 3, 4, 0, 5, 6, 7, 0};
    masklo = vld1_u8(masklo_data);
    maskhi = vld1_u8(maskhi_data);
  }

  // G2R2B1G1 R1B0G0R0 -> X1R1G1B1 X0R0G0B0
  uint8x8_t pxlo = vtbl1_u8(vget_low_u8(aSrc), masklo);
  // B3G3R3B2 G2R2B1G1 -> X3R3G3B3 X2R2G2B2
  uint8x8_t pxhi =
      vtbl1_u8(vext_u8(vget_low_u8(aSrc), vget_high_u8(aSrc), 4), maskhi);
  return vorrq_u8(vcombine_u8(pxlo, pxhi), alpha);
}

}  // namespace mozilla::gfx

#endif /* MOZILLA_GFX_SWIZZLE_NEON_H_ */