native.simd.ccm

// SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
module;
#include "native/isa_import.h"
#include <native/simd.h>
#include <native/integer.h>
#include <native/packing.h>
#include <native/simd/math/bits.h>
#include <native/targets.h>
export module native.simd;
export import native.numerics;
export import native.wide;
export import native.static_string;
export import native.types;
export import native.memory;
// Defaults use the owning module provider's baseline.
extern "C++" namespace native {
  template<class T, std::size_t N, isa<> Arch = NATIVE_BASELINE> struct simd;
}
// Keep the GMF partial specializations reachable to module consumers.
export namespace native { using ::native::mask_traits; }
#include "native/simd/exports.h"
#if NATIVE_HOST_X86
#define NATIVE_ARCH_REQUIRES(A) (::native::avx512_bf16 <= A)
#pragma clang attribute push(__attribute__((target("avx2,fma,avx512f,avx512dq,avx512bw,avx512vl,avx512bf16"))), apply_to=function)
export namespace native {
  namespace detail {
    template<std::size_t N,::native::isa<> A> requires NATIVE_ARCH_REQUIRES(A) &&(N==8 || N==16 || N==32)
    struct value_traits<simd<bf16,N,A>> {
      static constexpr isa<> value=avx512_bf16;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
  }
  template<std::size_t N, ::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch) &&(N == 8 || N == 16 || N == 32)
  struct simd<bf16,N,Arch> {
    using value_type = bf16;
    static constexpr isa<> architecture=Arch;
    using register_type = simd;
    using native_type = std::conditional_t<N == 8,__m128bh,std::conditional_t<N == 16,__m256bh,__m512bh>>;
    using bits_type = simd<std::uint16_t,N,architecture>;
    using mask = predicate<N,architecture>;
    using mask_type = mask;
    using predicate_type = mask;
    using vector_mask_type = simd<mask16,N,architecture>;
    template<class T> using rebind = simd<T,N,architecture>;
    static constexpr std::size_t lanes = N;
  private:
    native_type value_;
  public:
    simd() noexcept = default;
    native_inline constexpr explicit simd(bf16 value) noexcept
      : value_(__builtin_bit_cast(native_type,bits_type(value.to_bits()).to_native())) {}

    native_inline constexpr explicit simd(std::array<bf16,lanes> const & values) noexcept : simd(load(values.data())) {}
    template<class... T> requires(sizeof...(T) == lanes && (std::same_as<T,bf16> && ...))
    native_inline constexpr simd(T... values) noexcept : simd(std::array<bf16,lanes>{values...}) {}
    native_inline constexpr simd(native_type value) noexcept : value_(value) {}
    native_nodiscard native_inline constexpr operator native_type() const noexcept { return value_; }
    // Native interoperability must not add elementwise BF16 arithmetic.
#include "native/simd/bf16_reject_operators.h"
    native_nodiscard native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_nodiscard static native_inline constexpr simd from_native(native_type value) noexcept {
      simd result; result.value_ = value; return result;
    }

    native_nodiscard native_inline constexpr bits_type bits() const noexcept {
      return bits_type::from_native(__builtin_bit_cast(typename bits_type::native_type,value_));
    }

    native_nodiscard native_inline constexpr bits_type to_bits() const noexcept { return bits(); }
    native_nodiscard static native_inline constexpr simd from_bits(bits_type value) noexcept {
      return from_native(__builtin_bit_cast(native_type,value.to_native()));
    }

    native_nodiscard static native_inline constexpr simd load_bits(std::uint16_t const * p) noexcept {
      return from_bits(bits_type::load(p));
    }

    native_inline constexpr void store_bits(std::uint16_t * p) const noexcept { bits().store(p); }
    template<std::size_t Alignment = 1>
    native_nodiscard static native_inline constexpr simd load_memory(bf16 const * p) noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{};
        for (std::size_t i=0;i<lanes;++i) words[i]=p[i].to_bits();
        return load_bits(words.data());
      }
      native_type value; std::memcpy(&value, p, sizeof(value)); return from_native(value);
    }

    template<std::size_t Alignment = 1>
    native_inline constexpr void store_memory(bf16 * p) const noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<lanes;++i) p[i]=bf16::from_bits(words[i]);
        return;
      }
      std::memcpy(p, &value_, sizeof(value_));
    }

    native_nodiscard static native_inline constexpr simd load(bf16 const * p) noexcept { return load_memory(p); }
    native_inline constexpr void store(bf16 * p) const noexcept { store_memory(p); }
    native_nodiscard static native_inline constexpr simd loadu(bf16 const * p) noexcept { return load(p); }
    native_inline constexpr void storeu(bf16 * p) const noexcept { store(p); }
    native_nodiscard static native_inline constexpr simd load_partial(bf16 const * p, std::size_t n,
        bf16 fill = bf16::from_bits(0)) noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      std::array<bf16,lanes> values; values.fill(fill);
      if consteval {
        for (std::size_t i=0;i<n;++i) values[i]=p[i];
      } else { if (n) std::memcpy(values.data(), p, n * sizeof(bf16)); }
      return load(values.data());
    }

    native_inline constexpr void store_partial(bf16 * p, std::size_t n) const noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<n;++i) p[i]=bf16::from_bits(words[i]);
      } else { if (n) std::memcpy(p, &value_, n * sizeof(bf16)); }
    }
  };
 

  template<std::size_t N, ::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch) &&(N == 8 || N == 16 || N == 32)
  native_nodiscard native_inline constexpr simd<float,N/2,Arch> dot2(
      simd<bf16,N,Arch> a, simd<bf16,N,Arch> b,
      simd<float,N/2,Arch> accumulator) noexcept {
    if consteval { return detail::half_constant::dot2<false>(a,b,accumulator); }
    return simd<float,N/2,Arch>::from_native(
      detail::avx512_bf16_backend::dot2_native(a.to_native(), b.to_native(), accumulator.to_native()));
  }
}
 
#pragma clang attribute pop
#undef NATIVE_ARCH_REQUIRES
#define NATIVE_ARCH_REQUIRES(A) (::native::avx512_fp16 <= A)
#pragma clang attribute push(__attribute__((target("avx2,fma,avx512f,avx512dq,avx512bw,avx512vl,avx512fp16"))), apply_to=function)
export namespace native {
  namespace detail {
    template<::native::isa<> A> requires NATIVE_ARCH_REQUIRES(A)
    struct value_traits<simd<fp16,32,A>> {
      static constexpr isa<> value=avx512_fp16;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
  }
  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch) struct simd<fp16,32,Arch> {
    using value_type = fp16;
    static constexpr isa<> architecture=Arch;
    using register_type = simd;
    using native_type = __m512h;
    using bits_type = simd<std::uint16_t,32,architecture>;
    using mask = predicate<32,architecture>;
    using mask_type = mask;
    using predicate_type = mask;
    using vector_mask_type = simd<mask16,32,architecture>;
    template<class T> using rebind = simd<T,32,architecture>;
    static constexpr std::size_t lanes = 32;
  private:
    native_type value_;
  public:
    simd() noexcept = default;
    native_inline constexpr explicit simd(fp16 value) noexcept
      : value_(__builtin_bit_cast(native_type,bits_type(value.to_bits()).to_native())) {}

    native_inline constexpr explicit simd(std::array<fp16,lanes> const & values) noexcept : simd(load(values.data())) {}
    template<class... T> requires(sizeof...(T) == lanes && (std::same_as<T,fp16> && ...))
    native_inline constexpr simd(T... values) noexcept : simd(std::array<fp16,lanes>{values...}) {}
    native_inline constexpr simd(native_type value) noexcept : value_(value) {}
    native_nodiscard native_inline constexpr operator native_type() const noexcept { return value_; }
    native_nodiscard native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_nodiscard static native_inline constexpr simd from_native(native_type value) noexcept {
      simd result; result.value_ = value; return result;
    }

    native_nodiscard native_inline constexpr bits_type bits() const noexcept {
      return bits_type::from_native(__builtin_bit_cast(typename bits_type::native_type,value_));
    }

    native_nodiscard native_inline constexpr bits_type to_bits() const noexcept { return bits(); }
    native_nodiscard static native_inline constexpr simd from_bits(bits_type value) noexcept {
      return from_native(__builtin_bit_cast(native_type,value.to_native()));
    }

    native_nodiscard static native_inline constexpr simd load_bits(std::uint16_t const * p) noexcept {
      return from_bits(bits_type::load(p));
    }

    native_inline constexpr void store_bits(std::uint16_t * p) const noexcept { bits().store(p); }
    template<std::size_t Alignment = 1>
    native_nodiscard static native_inline constexpr simd load_memory(fp16 const * p) noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{};
        for (std::size_t i=0;i<lanes;++i) words[i]=p[i].to_bits();
        return load_bits(words.data());
      }
      native_type value; std::memcpy(&value, p, sizeof(value)); return from_native(value);
    }

    template<std::size_t Alignment = 1>
    native_inline constexpr void store_memory(fp16 * p) const noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<lanes;++i) p[i]=fp16::from_bits(words[i]);
        return;
      }
      std::memcpy(p, &value_, sizeof(value_));
    }

    native_nodiscard static native_inline constexpr simd load(fp16 const * p) noexcept { return load_memory(p); }
    native_inline constexpr void store(fp16 * p) const noexcept { store_memory(p); }
    native_nodiscard static native_inline constexpr simd loadu(fp16 const * p) noexcept { return load(p); }
    native_inline constexpr void storeu(fp16 * p) const noexcept { store(p); }
    native_nodiscard static native_inline constexpr simd load_partial(fp16 const * p, std::size_t n,
        fp16 fill = fp16::from_bits(0)) noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      std::array<fp16,lanes> values; values.fill(fill);
      if consteval {
        for (std::size_t i=0;i<n;++i) values[i]=p[i];
      } else { if (n) std::memcpy(values.data(), p, n * sizeof(fp16)); }
      return load(values.data());
    }

    native_inline constexpr void store_partial(fp16 * p, std::size_t n) const noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<n;++i) p[i]=fp16::from_bits(words[i]);
      } else { if (n) std::memcpy(p, &value_, n * sizeof(fp16)); }
    }

    native_nodiscard friend native_inline constexpr simd operator+(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::add,false>(a,b); }
      return from_native(detail::avx512_fp16_backend::add_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator-(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::subtract,false>(a,b); }
      return from_native(detail::avx512_fp16_backend::sub_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator*(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::multiply,false>(a,b); }
      return from_native(detail::avx512_fp16_backend::mul_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator/(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::divide,false>(a,b); }
      return from_native(detail::avx512_fp16_backend::div_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd sqrt(simd a) noexcept {
      if consteval { return detail::half_constant::square_root<false>(a); }
      return from_native(detail::avx512_fp16_backend::sqrt_half(a.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator-(simd a) noexcept {
      if consteval { return detail::half_constant::unary(a,[](auto x) { return std::uint16_t(x^0x8000); }); }
      return from_native(detail::avx512_fp16_backend::neg_half(a.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator==(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::equal_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::from_native(detail::avx512_fp16_backend::eq_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator!=(simd a,simd b) noexcept { return ~(a==b); }
    native_nodiscard friend native_inline constexpr mask operator<(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::less_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::from_native(detail::avx512_fp16_backend::lt_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator<=(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::less_bits<detail::constexpr_float::binary16>(x,y) || detail::constexpr_float::equal_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::from_native(detail::avx512_fp16_backend::le_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator>(simd a,simd b) noexcept { return b<a; }
    native_nodiscard friend native_inline constexpr mask operator>=(simd a,simd b) noexcept { return b<=a; }
    native_nodiscard friend native_inline constexpr simd select(mask m,simd a,simd b) noexcept {
      if consteval { return detail::half_constant::select(m,a,b); }
      return from_native(detail::avx512_fp16_backend::select_half(m.to_native(),a.value_,b.value_));
    }
  };
 

  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch)
  native_nodiscard native_inline constexpr simd<fp16,32,Arch> fma(
      simd<fp16,32,Arch> a,simd<fp16,32,Arch> b,simd<fp16,32,Arch> c) noexcept {
    if consteval { return detail::half_constant::fused<false>(a,b,c); }
    return simd<fp16,32,Arch>::from_native(
      detail::avx512_fp16_backend::fma_half(a.to_native(),b.to_native(),c.to_native()));
  }
}
 
#pragma clang attribute pop
#undef NATIVE_ARCH_REQUIRES
#endif
#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
#define NATIVE_ARCH_REQUIRES(A) (::native::neon_bf16 <= A)
#pragma clang attribute push(__attribute__((target("neon,bf16"))), apply_to=function)
export namespace native {
  namespace detail {
    template<::native::isa<> A> requires NATIVE_ARCH_REQUIRES(A)
    struct value_traits<simd<bf16,8,A>> {
      static constexpr isa<> value=neon_bf16;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
  }
  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch) struct simd<bf16,8,Arch> {
    using value_type = bf16;
    static constexpr isa<> architecture=Arch;
    using register_type = simd;
    using native_type = bfloat16x8_t;
    using bits_type = simd<std::uint16_t,8,architecture>;
    using mask = simd<mask16,8,architecture>;
    using mask_type = mask;
    using predicate_type = mask;
    using vector_mask_type = simd<mask16,8,architecture>;
    template<class T> using rebind = simd<T,8,architecture>;
    static constexpr std::size_t lanes = 8;
  private:
    native_type value_;
  public:
    simd() noexcept = default;
    native_inline constexpr explicit simd(bf16 value) noexcept
      : value_(__builtin_bit_cast(native_type,bits_type(value.to_bits()).to_native())) {}

    native_inline constexpr explicit simd(std::array<bf16,lanes> const & values) noexcept : simd(load(values.data())) {}
    template<class... T> requires(sizeof...(T) == lanes && (std::same_as<T,bf16> && ...))
    native_inline constexpr simd(T... values) noexcept : simd(std::array<bf16,lanes>{values...}) {}
    native_inline constexpr simd(native_type value) noexcept : value_(value) {}
    native_nodiscard native_inline constexpr operator native_type() const noexcept { return value_; }
    // Native interoperability must not add elementwise BF16 arithmetic.
#include "native/simd/bf16_reject_operators.h"
    native_nodiscard native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_nodiscard static native_inline constexpr simd from_native(native_type value) noexcept {
      simd result; result.value_ = value; return result;
    }

    native_nodiscard native_inline constexpr bits_type bits() const noexcept {
      return bits_type::from_native(__builtin_bit_cast(typename bits_type::native_type,value_));
    }

    native_nodiscard native_inline constexpr bits_type to_bits() const noexcept { return bits(); }
    native_nodiscard static native_inline constexpr simd from_bits(bits_type value) noexcept {
      return from_native(__builtin_bit_cast(native_type,value.to_native()));
    }

    native_nodiscard static native_inline constexpr simd load_bits(std::uint16_t const * p) noexcept {
      return from_bits(bits_type::load(p));
    }

    native_inline constexpr void store_bits(std::uint16_t * p) const noexcept { bits().store(p); }
    template<std::size_t Alignment = 1>
    native_nodiscard static native_inline constexpr simd load_memory(bf16 const * p) noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{};
        for (std::size_t i=0;i<lanes;++i) words[i]=p[i].to_bits();
        return load_bits(words.data());
      }
      native_type value; std::memcpy(&value, p, sizeof(value)); return from_native(value);
    }

    template<std::size_t Alignment = 1>
    native_inline constexpr void store_memory(bf16 * p) const noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<lanes;++i) p[i]=bf16::from_bits(words[i]);
        return;
      }
      std::memcpy(p, &value_, sizeof(value_));
    }

    native_nodiscard static native_inline constexpr simd load(bf16 const * p) noexcept { return load_memory(p); }
    native_inline constexpr void store(bf16 * p) const noexcept { store_memory(p); }
    native_nodiscard static native_inline constexpr simd loadu(bf16 const * p) noexcept { return load(p); }
    native_inline constexpr void storeu(bf16 * p) const noexcept { store(p); }
    native_nodiscard static native_inline constexpr simd load_partial(bf16 const * p, std::size_t n,
        bf16 fill = bf16::from_bits(0)) noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      std::array<bf16,lanes> values; values.fill(fill);
      if consteval {
        for (std::size_t i=0;i<n;++i) values[i]=p[i];
      } else { if (n) std::memcpy(values.data(), p, n * sizeof(bf16)); }
      return load(values.data());
    }

    native_inline constexpr void store_partial(bf16 * p, std::size_t n) const noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<n;++i) p[i]=bf16::from_bits(words[i]);
      } else { if (n) std::memcpy(p, &value_, n * sizeof(bf16)); }
    }
  };
 

  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch)
  native_nodiscard native_inline constexpr simd<float,4,Arch> dot2(
      simd<bf16,8,Arch> a, simd<bf16,8,Arch> b,
      simd<float,4,Arch> accumulator) noexcept {
    if consteval { return detail::half_constant::dot2<true>(a,b,accumulator); }
    return simd<float,4,Arch>::from_native(
      detail::neon_bf16_backend::dot2_native(a.to_native(), b.to_native(), accumulator.to_native()));
  }
}
 
#pragma clang attribute pop
#undef NATIVE_ARCH_REQUIRES
#define NATIVE_ARCH_REQUIRES(A) (::native::neon_fp16 <= A)
#pragma clang attribute push(__attribute__((target("neon,fullfp16"))), apply_to=function)
export namespace native {
  namespace detail {
    template<::native::isa<> A> requires NATIVE_ARCH_REQUIRES(A)
    struct value_traits<simd<fp16,8,A>> {
      static constexpr isa<> value=neon_fp16;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
  }
  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch) struct simd<fp16,8,Arch> {
    using value_type = fp16;
    static constexpr isa<> architecture=Arch;
    using register_type = simd;
    using native_type = float16x8_t;
    using bits_type = simd<std::uint16_t,8,architecture>;
    using mask = simd<mask16,8,architecture>;
    using mask_type = mask;
    using predicate_type = mask;
    using vector_mask_type = simd<mask16,8,architecture>;
    template<class T> using rebind = simd<T,8,architecture>;
    static constexpr std::size_t lanes = 8;
  private:
    native_type value_;
  public:
    simd() noexcept = default;
    native_inline constexpr explicit simd(fp16 value) noexcept
      : value_(__builtin_bit_cast(native_type,bits_type(value.to_bits()).to_native())) {}

    native_inline constexpr explicit simd(std::array<fp16,lanes> const & values) noexcept : simd(load(values.data())) {}
    template<class... T> requires(sizeof...(T) == lanes && (std::same_as<T,fp16> && ...))
    native_inline constexpr simd(T... values) noexcept : simd(std::array<fp16,lanes>{values...}) {}
    native_inline constexpr simd(native_type value) noexcept : value_(value) {}
    native_nodiscard native_inline constexpr operator native_type() const noexcept { return value_; }
    native_nodiscard native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_nodiscard static native_inline constexpr simd from_native(native_type value) noexcept {
      simd result; result.value_ = value; return result;
    }

    native_nodiscard native_inline constexpr bits_type bits() const noexcept {
      return bits_type::from_native(__builtin_bit_cast(typename bits_type::native_type,value_));
    }

    native_nodiscard native_inline constexpr bits_type to_bits() const noexcept { return bits(); }
    native_nodiscard static native_inline constexpr simd from_bits(bits_type value) noexcept {
      return from_native(__builtin_bit_cast(native_type,value.to_native()));
    }

    native_nodiscard static native_inline constexpr simd load_bits(std::uint16_t const * p) noexcept {
      return from_bits(bits_type::load(p));
    }

    native_inline constexpr void store_bits(std::uint16_t * p) const noexcept { bits().store(p); }
    template<std::size_t Alignment = 1>
    native_nodiscard static native_inline constexpr simd load_memory(fp16 const * p) noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{};
        for (std::size_t i=0;i<lanes;++i) words[i]=p[i].to_bits();
        return load_bits(words.data());
      }
      native_type value; std::memcpy(&value, p, sizeof(value)); return from_native(value);
    }

    template<std::size_t Alignment = 1>
    native_inline constexpr void store_memory(fp16 * p) const noexcept {
      static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<lanes;++i) p[i]=fp16::from_bits(words[i]);
        return;
      }
      std::memcpy(p, &value_, sizeof(value_));
    }

    native_nodiscard static native_inline constexpr simd load(fp16 const * p) noexcept { return load_memory(p); }
    native_inline constexpr void store(fp16 * p) const noexcept { store_memory(p); }
    native_nodiscard static native_inline constexpr simd loadu(fp16 const * p) noexcept { return load(p); }
    native_inline constexpr void storeu(fp16 * p) const noexcept { store(p); }
    native_nodiscard static native_inline constexpr simd load_partial(fp16 const * p, std::size_t n,
        fp16 fill = fp16::from_bits(0)) noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      std::array<fp16,lanes> values; values.fill(fill);
      if consteval {
        for (std::size_t i=0;i<n;++i) values[i]=p[i];
      } else { if (n) std::memcpy(values.data(), p, n * sizeof(fp16)); }
      return load(values.data());
    }

    native_inline constexpr void store_partial(fp16 * p, std::size_t n) const noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n <= lanes);
      if consteval {
        std::array<std::uint16_t,lanes> words{}; store_bits(words.data());
        for (std::size_t i=0;i<n;++i) p[i]=fp16::from_bits(words[i]);
      } else { if (n) std::memcpy(p, &value_, n * sizeof(fp16)); }
    }

    native_nodiscard friend native_inline constexpr simd operator+(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::add,true>(a,b); }
      return from_native(detail::neon_fp16_backend::add_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator-(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::subtract,true>(a,b); }
      return from_native(detail::neon_fp16_backend::sub_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator*(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::multiply,true>(a,b); }
      return from_native(detail::neon_fp16_backend::mul_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator/(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::arithmetic<detail::half_constant::operation::divide,true>(a,b); }
      return from_native(detail::neon_fp16_backend::div_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr simd sqrt(simd a) noexcept {
      if consteval { return detail::half_constant::square_root<true>(a); }
      return from_native(detail::neon_fp16_backend::sqrt_half(a.value_));
    }

    native_nodiscard friend native_inline constexpr simd operator-(simd a) noexcept {
      if consteval { return detail::half_constant::unary(a,[](auto x) { return std::uint16_t(x^0x8000); }); }
      return from_native(detail::neon_fp16_backend::neg_half(a.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator==(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::equal_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::unsafe_from_native(detail::neon_fp16_backend::eq_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator!=(simd a,simd b) noexcept { return ~(a==b); }
    native_nodiscard friend native_inline constexpr mask operator<(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::less_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::unsafe_from_native(detail::neon_fp16_backend::lt_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator<=(simd a,simd b) noexcept {
      if consteval { return detail::half_constant::compare(a,b,[](auto x,auto y) { return detail::constexpr_float::less_bits<detail::constexpr_float::binary16>(x,y) || detail::constexpr_float::equal_bits<detail::constexpr_float::binary16>(x,y); }); }
      return mask::unsafe_from_native(detail::neon_fp16_backend::le_half(a.value_,b.value_));
    }

    native_nodiscard friend native_inline constexpr mask operator>(simd a,simd b) noexcept { return b<a; }
    native_nodiscard friend native_inline constexpr mask operator>=(simd a,simd b) noexcept { return b<=a; }
    native_nodiscard friend native_inline constexpr simd select(mask m,simd a,simd b) noexcept {
      if consteval { return detail::half_constant::select(m,a,b); }
      return from_native(detail::neon_fp16_backend::select_half(m.to_native(),a.value_,b.value_));
    }
  };
 

  template<::native::isa<> Arch> requires NATIVE_ARCH_REQUIRES(Arch)
  native_nodiscard native_inline constexpr simd<fp16,8,Arch> fma(
      simd<fp16,8,Arch> a,simd<fp16,8,Arch> b,simd<fp16,8,Arch> c) noexcept {
    if consteval { return detail::half_constant::fused<true>(a,b,c); }
    return simd<fp16,8,Arch>::from_native(
      detail::neon_fp16_backend::fma_half(a.to_native(),b.to_native(),c.to_native()));
  }
}
 
#pragma clang attribute pop
#undef NATIVE_ARCH_REQUIRES
#endif
// SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
// Register shapes needed by instruction interfaces outside the arithmetic kernels.
#if !NATIVE_HOST_WASM
export namespace native {
  namespace detail {
    template<class T> concept instruction_element = simd_integer_element<T> ||
      std::same_as<T,float> || std::same_as<T,double> ||
      std::same_as<T,fp16> || std::same_as<T,bf16>;
 
    template<class T, std::size_t N, isa<> A>
    inline constexpr bool instruction_storage_shape = [] {
      if constexpr(!instruction_element<T> || N<2) return false;
      else if constexpr(N>64/sizeof(T)) return false;
      else {
        constexpr auto bytes=sizeof(T)*N;
#if NATIVE_HOST_NEON
        if constexpr(!(neon<=A)) return false;
        if constexpr(std::same_as<T,fp16>) return N==4 || (N==8 && !(neon_fp16<=A));
        if constexpr(std::same_as<T,bf16>) return N==4 || (N==8 && !(neon_bf16<=A));
        if constexpr(std::same_as<T,double>) return N==2;
        // Existing two-lane 32-bit vectors have intentional four-lane storage.
        return simd_integer_element<T> && sizeof(T)<4 && bytes==8;
#elif NATIVE_HOST_X86
        if constexpr(!A.has(x86_feature::sse2)) return false;
        if constexpr(std::same_as<T,fp16>) return N==4 || N==8;
        if constexpr(std::same_as<T,bf16>) return N==4 || (N==8 && !(avx512_bf16<=A));
        if constexpr(bytes==8) return simd_integer_element<T> && sizeof(T)<4;
        if constexpr(bytes==16 || bytes==32) {
          if constexpr(bytes==32 && !A.has(x86_feature::avx)) return false;
          return std::same_as<T,double> || !(avx2<=A);
        }
        if constexpr(bytes==64 && A.has(x86_feature::avx512f)) {
          if constexpr(std::same_as<T,double>) return true;
          return !(kernel_base<=A) || (sizeof(T)<4 && !A.has(x86_feature::avx512bw));
        }
        return false;
#else
        return false;
#endif
      }
    }();
 
    template<std::size_t N,isa<> A> inline constexpr bool instruction_predicate_shape =
      N>0 && N<=64 &&
#if NATIVE_HOST_X86
      A.has(x86_feature::sse2) && !((kernel_base<=A) &&
        (N==1 || N==2 || N==3 || N==4 || N==8 || N==16 ||
          ((N==32 || N==64) && A.has(x86_feature::avx512bw))));
#elif NATIVE_HOST_NEON
      (neon<=A);
#else
      false;
#endif
 
    template<class T,std::size_t N,isa<> A>
      requires ordinary_simd_element<T> && instruction_storage_shape<T,N,A>
    struct value_traits<simd<T,N,A>> {
      static constexpr isa<> value=A;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
    template<std::size_t N,isa<> A> requires instruction_predicate_shape<N,A>
    struct value_traits<predicate<N,A>> {
      static constexpr isa<> value=A;
      static constexpr bool known=true;
      static constexpr bool aggregate_default=false;
    };
 
    template<class T, std::size_t N> struct instruction_register {
#if NATIVE_HOST_NEON
      using type=std::conditional_t<std::same_as<T,double>,float64x2_t,
        std::conditional_t<(sizeof(T)*N<=8),uint8x8_t,uint8x16_t>>;
#elif NATIVE_HOST_X86
      using type=std::conditional_t<std::same_as<T,float>,
        std::conditional_t<(sizeof(T)*N<=16),__m128,std::conditional_t<(sizeof(T)*N==32),__m256,__m512>>,
        std::conditional_t<std::same_as<T,double>,
          std::conditional_t<(sizeof(T)*N<=16),__m128d,std::conditional_t<(sizeof(T)*N==32),__m256d,__m512d>>,
          std::conditional_t<(sizeof(T)*N<=16),__m128i,std::conditional_t<(sizeof(T)*N==32),__m256i,__m512i>>>>;
#endif
    };
  }

  template<std::size_t N, isa<> A> requires detail::instruction_predicate_shape<N,A>
  struct predicate<N,A> {
    static constexpr isa<> architecture=A;
    static constexpr std::size_t lanes=N;
    using native_type=std::conditional_t<(N<=8),std::uint8_t,
      std::conditional_t<(N<=16),std::uint16_t,std::conditional_t<(N<=32),std::uint32_t,std::uint64_t>>>;
    using mask=predicate;
    using mask_type=predicate;
    static constexpr bool compact=true;
  private:
    native_type value_{};
    static constexpr std::uint64_t active=[] { if constexpr(N==64) return ~std::uint64_t{}; else return (std::uint64_t{1}<<N)-1; }();
  public:
    native_inline constexpr predicate() noexcept=default;
    explicit native_inline constexpr predicate(bool value) noexcept : value_(value?native_type(active):0) {}
    static native_inline constexpr predicate from_bits(std::uint64_t bits) noexcept { predicate p; p.value_=native_type(bits&active); return p; }
    static native_inline constexpr predicate from_bitset(std::uint64_t bits) noexcept { return from_bits(bits); }
    static native_inline constexpr predicate from_native(native_type bits) noexcept { return from_bits(bits); }
    native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_inline constexpr std::uint64_t bits() const noexcept { return value_; }
    native_inline constexpr std::uint64_t to_bitset() const noexcept { return value_; }
    friend native_inline constexpr bool any(predicate p) noexcept { return p.value_!=0; }
    friend native_inline constexpr bool all(predicate p) noexcept { return p.value_==active; }
    friend native_inline constexpr bool none(predicate p) noexcept { return p.value_==0; }
    friend native_inline constexpr predicate operator~(predicate p) noexcept { return from_bits(~p.value_); }
    friend native_inline constexpr predicate operator!(predicate p) noexcept { return ~p; }
    friend native_inline constexpr predicate operator&(predicate a,predicate b) noexcept { return from_bits(a.value_&b.value_); }
    friend native_inline constexpr predicate operator|(predicate a,predicate b) noexcept { return from_bits(a.value_|b.value_); }
    friend native_inline constexpr predicate operator^(predicate a,predicate b) noexcept { return from_bits(a.value_^b.value_); }
    friend native_inline constexpr predicate operator==(predicate a,predicate b) noexcept { return ~(a^b); }
    friend native_inline constexpr predicate operator!=(predicate a,predicate b) noexcept { return a^b; }
    native_inline constexpr predicate & operator&=(predicate b) noexcept { return *this=*this&b; }
    native_inline constexpr predicate & operator|=(predicate b) noexcept { return *this=*this|b; }
    native_inline constexpr predicate & operator^=(predicate b) noexcept { return *this=*this^b; }
    friend native_inline constexpr predicate select(predicate p,predicate a,predicate b) noexcept { return (p&a)|(~p&b); }
  };

  template<class T, std::size_t N, isa<> A> requires detail::instruction_storage_shape<T,N,A>
  struct alignas(typename detail::instruction_register<T,N>::type) simd<T,N,A> {
    using value_type=T;
    using register_type=simd;
    using native_type=typename detail::instruction_register<T,N>::type;
    static constexpr isa<> architecture=A;
    static constexpr std::size_t lanes=N;
    using mask=predicate<N,A>;
    using mask_type=mask;
    using predicate_type=mask;
    using bits_type=simd<std::conditional_t<sizeof(T)==1,std::uint8_t,
      std::conditional_t<sizeof(T)==2,std::uint16_t,
        std::conditional_t<sizeof(T)==4,std::uint32_t,std::uint64_t>>>,N,A>;
    template<class U> using rebind=simd<U,N,A>;
  private:
    native_type value_;
  public:
    // Clang may accept a storage record mixed with a builtin vector even
    // without a native conversion operator. Reject half arithmetic explicitly.
#define NATIVE_INSTRUCTION_HALF_REJECT_BINARY(OP) \
 \
    friend void operator OP(simd,simd) \
      requires(std::same_as<T,fp16> || std::same_as<T,bf16>) = delete; \
 \
    template<class U> requires((std::same_as<T,fp16> || std::same_as<T,bf16>) && \
      !std::same_as<std::remove_cvref_t<U>,simd>) \
    friend void operator OP(simd,U) = delete; \
 \
    template<class U> requires((std::same_as<T,fp16> || std::same_as<T,bf16>) && \
      !std::same_as<std::remove_cvref_t<U>,simd>) \
    friend void operator OP(U,simd) = delete;
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(+)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(-)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(*)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(/)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(%)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(&)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(|)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(^)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(<<)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(>>)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(==)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(!=)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(<)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(<=)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(>)
    NATIVE_INSTRUCTION_HALF_REJECT_BINARY(>=)
#undef NATIVE_INSTRUCTION_HALF_REJECT_BINARY
#define NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(OP) \
 \
    template<class U> requires(std::same_as<T,fp16> || std::same_as<T,bf16>) \
    void operator OP(U) = delete;
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(+=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(-=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(*=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(/=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(%=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(&=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(|=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(^=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(<<=)
    NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN(>>=)
#undef NATIVE_INSTRUCTION_HALF_REJECT_ASSIGN
    friend void operator+(simd) requires(std::same_as<T,fp16> || std::same_as<T,bf16>) = delete;
    friend void operator-(simd) requires(std::same_as<T,fp16> || std::same_as<T,bf16>) = delete;
    friend void operator~(simd) requires(std::same_as<T,fp16> || std::same_as<T,bf16>) = delete;
    friend void operator!(simd) requires(std::same_as<T,fp16> || std::same_as<T,bf16>) = delete;
    constexpr simd() noexcept=default;
    native_inline constexpr explicit simd(T value) noexcept {
      std::array<T,N> values;
      values.fill(value);
      *this=load(values.data());
    }

    native_inline constexpr explicit simd(std::array<T,N> const & values) noexcept : simd(load(values.data())) {}
    template<class... U> requires(sizeof...(U)==N && (std::same_as<U,T> && ...))
    native_inline constexpr simd(U... values) noexcept : simd(std::array<T,N>{values...}) {}
#if NATIVE_HOST_X86
    // Native vector arguments and returns must carry their register ABI even
    // when an always-inline caller has already enabled that target.
    native_nodiscard native_inline constexpr native_target("sse2")
    native_type to_native() const noexcept requires(sizeof(native_type)==16) { return value_; }
    native_nodiscard static native_inline constexpr native_target("sse2")
    simd from_native(native_type value) noexcept requires(sizeof(native_type)==16) {
      simd result; result.value_=value; return result;
    }

    native_nodiscard static native_inline constexpr native_target("sse2")
    simd unsafe_from_native(native_type value) noexcept requires(sizeof(native_type)==16) {
      simd result; result.value_=value; return result;
    }

    native_nodiscard native_inline constexpr native_target("avx")
    native_type to_native() const noexcept requires(sizeof(native_type)==32) { return value_; }
    native_nodiscard static native_inline constexpr native_target("avx")
    simd from_native(native_type value) noexcept requires(sizeof(native_type)==32) {
      simd result; result.value_=value; return result;
    }

    native_nodiscard static native_inline constexpr native_target("avx")
    simd unsafe_from_native(native_type value) noexcept requires(sizeof(native_type)==32) {
      simd result; result.value_=value; return result;
    }

    native_nodiscard native_inline constexpr native_target("avx512f")
    native_type to_native() const noexcept requires(sizeof(native_type)==64) { return value_; }
    native_nodiscard static native_inline constexpr native_target("avx512f")
    simd from_native(native_type value) noexcept requires(sizeof(native_type)==64) {
      simd result; result.value_=value; return result;
    }

    native_nodiscard static native_inline constexpr native_target("avx512f")
    simd unsafe_from_native(native_type value) noexcept requires(sizeof(native_type)==64) {
      simd result; result.value_=value; return result;
    }
#else
    native_nodiscard native_inline constexpr native_type to_native() const noexcept { return value_; }
    native_nodiscard static native_inline constexpr simd from_native(native_type value) noexcept {
      simd result; result.value_=value; return result;
    }
#endif
    template<std::size_t Alignment=1>
    native_nodiscard static native_inline constexpr simd load_memory(T const * p) noexcept {
      static_assert(Alignment>0 && (Alignment&(Alignment-1))==0);
      simd result{};
      if consteval {
        std::array<T,sizeof(native_type)/sizeof(T)> values{};
        for(std::size_t i=0;i<N;++i) values[i]=p[i];
        result.value_=std::bit_cast<native_type>(values);
      } else { std::memcpy(&result.value_,p,sizeof(T)*N); }
      return result;
    }

    template<std::size_t Alignment=1>
    native_inline constexpr void store_memory(T * p) const noexcept {
      static_assert(Alignment>0 && (Alignment&(Alignment-1))==0);
      if consteval {
        auto values=std::bit_cast<std::array<T,sizeof(native_type)/sizeof(T)>>(value_);
        for(std::size_t i=0;i<N;++i) p[i]=values[i];
      } else { std::memcpy(p,&value_,sizeof(T)*N); }
    }

    native_nodiscard static native_inline constexpr simd load(T const * p) noexcept { return load_memory(p); }
    native_nodiscard static native_inline constexpr simd loadu(T const * p) noexcept { return load(p); }
    native_inline constexpr void store(T * p) const noexcept { store_memory(p); }
    native_inline constexpr void storeu(T * p) const noexcept { store(p); }
    native_nodiscard static native_inline constexpr simd load_partial(T const * p,std::size_t n,T fill=T{}) noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n<=N);
      std::array<T,N> values; values.fill(fill);
      if consteval {
        for(std::size_t i=0;i<n;++i) values[i]=p[i];
      } else { if(n) std::memcpy(values.data(),p,n*sizeof(T)); }
      return load(values.data());
    }

    native_inline constexpr void store_partial(T * p,std::size_t n) const noexcept native_diagnose_if(n > simd::lanes,"partial SIMD count exceeds the lane count") {
      assert(n<=N);
      if consteval {
        if(n) {
          auto values=std::bit_cast<std::array<T,sizeof(native_type)/sizeof(T)>>(value_);
          for(std::size_t i=0;i<n;++i) p[i]=values[i];
        }
      } else { if(n) std::memcpy(p,&value_,n*sizeof(T)); }
    }

    native_nodiscard native_inline constexpr bits_type bits() const noexcept requires(std::same_as<T,fp16> || std::same_as<T,bf16>) {
      return bits_type::from_native(std::bit_cast<typename bits_type::native_type>(value_));
    }

    native_nodiscard native_inline constexpr bits_type to_bits() const noexcept requires(std::same_as<T,fp16> || std::same_as<T,bf16>) { return bits(); }
    native_nodiscard static native_inline constexpr simd from_bits(bits_type words) noexcept requires(std::same_as<T,fp16> || std::same_as<T,bf16>) {
      return from_native(std::bit_cast<native_type>(words.to_native()));
    }

    native_nodiscard static native_inline constexpr simd load_bits(std::uint16_t const * p) noexcept requires(std::same_as<T,fp16> || std::same_as<T,bf16>) {
      if consteval { return from_bits(bits_type::load(p)); }
      else { simd result{}; std::memcpy(&result.value_,p,2*N); return result; }
    }

    native_inline constexpr void store_bits(std::uint16_t * p) const noexcept requires(std::same_as<T,fp16> || std::same_as<T,bf16>) {
      if consteval { bits().store(p); }
      else { std::memcpy(p,&value_,2*N); }
    }
  };
}
 
#endif