simd.h

// SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
#pragma once
// Included below the FTZ module declaration; all SIMD operations are dependent.
 
namespace ftz::detail {
  // An emulation-only ISA has no register operations to prefer. Reuse the
  // scalar semantic graph; adding polyfill to a native ISA keeps its fast path.
  template<class V> inline constexpr bool ftz32_emulated = [] {
    if constexpr (requires { V::architecture; })
      return V::architecture == ::native::isa<>(::native::polyfill);
    else return false;
  }();
  using ::native::mask_bits;
  template <class V> using ftz32_bridge = native::detail::fp32_bit_bridge<V>;
  template <class V> using ftz32_words = typename V::bits_type;
  template <auto Operation, class V, class... X>
  [[nodiscard]] constexpr V ftz32_constant(V input, X... rest) noexcept {
    using B = ftz32_bridge<V>; using U = ftz32_words<V>;
    std::array<std::array<std::uint32_t,V::lanes>,1+sizeof...(X)> words{};
    B::encode(input).store(words[0].data());
    std::size_t index=1;
    (B::encode(rest).store(words[index++].data()),...);
    std::array<std::uint32_t,V::lanes> result{};
    for(std::size_t lane=0;lane<V::lanes;++lane)
      result[lane]=[&]<std::size_t... I>(std::index_sequence<I...>) {
        return Operation(words[I][lane]...);
      }(std::make_index_sequence<1+sizeof...(X)>{});
    return B::decode(U::load(result.data()));
  }
  template <class U> [[nodiscard]] native_inline constexpr bool ftz32_any(U mask) noexcept {
    return any(mask != U(0));
  }
  template <class U> [[nodiscard]] native_inline constexpr native_const U ftz32_import_words(U bits) noexcept {
    auto magnitude = bits & U(0x7fffffffu);
    auto tiny = U(0x00800000u) > magnitude;
    return select(tiny, bits & U(0x80000000u), bits);
  }
  template <class V> [[nodiscard]] native_inline constexpr native_const ftz32_words<V> ftz32_nonfinite(V v) noexcept {
    using U = ftz32_words<V>;
    return mask_bits<std::uint32_t>(U(ftz32_infinity) > (ftz32_bridge<V>::encode(v) & U(0x7fffffffu))) ^ U(0xffffffffu);
  }
  template <class V>
  [[nodiscard]] native_inline constexpr native_const ftz32_words<V> ftz32_repair_mask(V result) noexcept {
    using U = ftz32_words<V>;
    auto boundary_mask = mask_bits<std::uint32_t>((ftz32_bridge<V>::encode(result) & U(0x7fffffffu)) > U(0x00800000u)) ^ U(0xffffffffu);
    return boundary_mask;
  }
  template <auto Repair, class A, std::size_t... I>
  [[nodiscard]] native_inline constexpr native_pure std::uint32_t ftz32_repair_lane(A const & inputs, std::size_t lane, std::index_sequence<I...>) noexcept {
    return Repair(inputs[I][lane]...);
  }
  // No lane extraction on the normal path. Only flagged lanes call the shared
  // scalar contract; the operation and backend are compile-time selections.
  template <auto Repair, class V, class... X>
  [[nodiscard]] native_inline constexpr native_const V ftz32_repair(V result, ftz32_words<V> mask, X... operands) noexcept {
    if (!ftz32_any(mask)) return result;
    using B = ftz32_bridge<V>;
    using U = ftz32_words<V>;
    std::array<std::uint32_t, V::lanes> flags, output;
    std::array<std::array<std::uint32_t, V::lanes>, sizeof...(X)> inputs;
    mask.store(flags.data()); B::encode(result).store(output.data());
    std::size_t index = 0;
    (B::encode(operands).store(inputs[index++].data()), ...);
    for (std::size_t lane = 0; lane < V::lanes; ++lane)
      if (flags[lane] != 0)
        output[lane] = ftz32_repair_lane<Repair>(inputs, lane, std::index_sequence_for<X...>{});
    return B::decode(U::load(output.data()));
  }
  // General FTZ scaling owns its semantic graph: raw native scaling exists
  // only where the target has an instruction. Normal bases need no arithmetic
  // on their significands, including the sole tiny result that rounds to normal.
  template <class V>
  [[nodiscard]] native_inline constexpr native_const V ftz32_vector_scaleb(V value, V exponent) noexcept {
    using B = ftz32_bridge<V>; using U = ftz32_words<V>;
    using I = typename V::template rebind<std::int32_t>;
    U x = B::encode(value), y = ftz32_import_words(B::encode(exponent));
    U magnitude = x & U(0x7fffffffu), shift_magnitude = y & U(0x7fffffffu);
    U sign = x & U(0x80000000u), fraction = x & U(0x007fffffu);
    // +/-256 settles every finite FTZ base. Clamp by word magnitude before
    // floor/conversion, so even infinities and NaNs have a representable integer
    // intermediate. Their value semantics are selected independently below.
    U bounded = select(shift_magnitude < U(0x43800000u), y,
      (y & U(0x80000000u)) | U(0x43800000u));
    I shift = convert<std::int32_t>(floor(B::decode(bounded)));
    U biased = magnitude.template right<23>();
    I adjusted = I::from_native(__builtin_bit_cast(typename I::native_type, biased.to_native())) + shift;
    U fields = U::from_native(__builtin_bit_cast(typename U::native_type, adjusted.to_native()));
    U result = select((adjusted > I(0)) & (adjusted < I(255)),
      fields.template left<23>() | fraction,
      select(adjusted > I(254), U(0x7f800000u), U(0)));
    // A maximum significand at adjusted exponent zero is exactly halfway
    // below minimum normal; nearest-even rounds upward before signed flushing.
    result = select((adjusted == I(0)) & (fraction == U(0x007fffffu)), U(0x00800000u), result) | sign;
    result = select((magnitude == U(0)) | (magnitude == U(0x7f800000u)), x, result);
    result = select(magnitude > U(0x7f800000u), x | U(0x00400000u), result);
    U upward = select(magnitude == U(0), U(0x7fc00000u), sign | U(0x7f800000u));
    U downward = select(magnitude == U(0x7f800000u), U(0x7fc00000u), sign);
    // Infinite exponents treat every NaN base as quiet: +infinity gives
    // +infinity, -infinity gives +zero, independent of NaN sign/payload.
    upward = select(magnitude > U(0x7f800000u), U(0x7f800000u), upward);
    downward = select(magnitude > U(0x7f800000u), U(0), downward);
    result = select(y == U(0x7f800000u), upward, result);
    result = select(y == U(0xff800000u), downward, result);
    return B::decode(select(shift_magnitude > U(0x7f800000u), y | U(0x00400000u), result));
  }
  template <bool Hardware, class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_add(V a, V b) noexcept {
    if consteval { return ftz32_constant<ftz32_add<Hardware>>(a, b); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_add<Hardware>>(a, b);
    V r = a + b;
    if constexpr(Hardware) return r;
    else return ftz32_repair<ftz32_add<Hardware>>(r, ftz32_repair_mask(r), a, b);
  }
  template <bool Hardware, class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_sub(V a, V b) noexcept {
    if consteval { return ftz32_constant<ftz32_sub<Hardware>>(a, b); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_sub<Hardware>>(a, b);
    V r = a - b;
    if constexpr(Hardware) return r;
    else return ftz32_repair<ftz32_sub<Hardware>>(r, ftz32_repair_mask(r), a, b);
  }
  template <class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_mul(V a, V b) noexcept {
    if consteval { return ftz32_constant<ftz32_mul>(a, b); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_mul>(a, b);
    V r = a * b; return ftz32_repair<ftz32_mul>(r, ftz32_repair_mask(r), a, b);
  }
  template <class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_fma(V a, V b, V c) noexcept {
    if consteval { return ftz32_constant<ftz32_fma>(a, b, c); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_fma>(a, b, c);
    V r = fma(a, b, c); return ftz32_repair<ftz32_fma>(r, ftz32_repair_mask(r), a, b, c);
  }
  // Same normalized three-refinement graph as ftz/math/approx.h.
  // Every mantissa operation stays normal. Integer exponent scaling handles
  // ordinary results; unusual scales and IEEE-like special values use the core.
  template <bool Hardware, class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_div(V a, V b) noexcept {
    if consteval { return ftz32_constant<ftz32_div<Hardware>>(a, b); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_div<Hardware>>(a, b);
    using B = ftz32_bridge<V>; using U = ftz32_words<V>;
    U aw = B::encode(a), bw = B::encode(b);
    U aa = aw & U(0x7fffffffu), bb = bw & U(0x7fffffffu);
    U ma = U(0x3f800000u) | (aa & U(0x007fffffu));
    U mb = U(0x3f800000u) | (bb & U(0x007fffffu));
    V m = B::decode(mb), r = B::decode(U(0x7ef311c3u) - mb);
    V e = fma(-m, r, V(1)); r = fma(r, e, r);
    e = fma(-m, r, V(1)); r = fma(r, e, r);
    e = fma(-m, r, V(1)); r = fma(r, e, r);
    r = B::decode(ma) * r;
    U rw = B::encode(r);
    U exponent = rw.template right<23>() + aa.template right<23>() - bb.template right<23>();
    U normal = mask_bits<std::uint32_t>((exponent > U(0)) & (U(255) > exponent));
    U bits = ((aw ^ bw) & U(0x80000000u)) | exponent.template left<23>() | (rw & U(0x007fffffu));
    U exceptional = (normal ^ U(0xffffffffu)) | mask_bits<std::uint32_t>((aa == U(0)) | (bb == U(0))) |
                    ftz32_nonfinite(a) | ftz32_nonfinite(b);
    return ftz32_repair<ftz32_div<Hardware>>(B::decode(bits), exceptional, a, b);
  }
  template <class V> [[nodiscard]] native_inline constexpr native_const V ftz32_vector_sqrt(V a) noexcept {
    if consteval { return ftz32_constant<ftz32_sqrt>(a); }
    if constexpr (ftz32_emulated<V>) return ftz32_constant<ftz32_sqrt>(a);
    using B = ftz32_bridge<V>; using U = ftz32_words<V>;
    U aw = B::encode(a), aa = aw & U(0x7fffffffu), exponent_a = aa.template right<23>();
    U parity = U(1) - (exponent_a & U(1));
    U mword = (U(0x3f800000u) | (aa & U(0x007fffffu))) + parity.template left<23>();
    V m = B::decode(mword), r = B::decode(U(0x5f375a86u) - mword.template right<1>());
    V p = m * r, e = fma(-p, r, V(1)), h = V(0.5f) * r; r = fma(h, e, r);
    p = m * r; e = fma(-p, r, V(1)); h = V(0.5f) * r; r = fma(h, e, r);
    p = m * r; e = fma(-p, r, V(1)); h = V(0.5f) * r; r = fma(h, e, r);
    r = m * r;
    U rw = B::encode(r);
    U exponent = rw.template right<23>() + (exponent_a + U(127) - parity).template right<1>() - U(127);
    U bits = exponent.template left<23>() | (rw & U(0x007fffffu));
    U exceptional = mask_bits<std::uint32_t>((aa == U(0)) | ((aw & U(0x80000000u)) > U(0))) | ftz32_nonfinite(a);
    return ftz32_repair<ftz32_sqrt>(B::decode(bits), exceptional, a);
  }
 
  template <class V> concept raw_register = requires { typename V::value_type; typename V::bits_type; V::lanes; } && std::same_as<typename V::value_type,float>;
  template <class R> concept ftz32_vector = requires { typename R::value_type; typename R::register_type; } && ftz32_type<typename R::value_type>;
  template <class V, class F = ftz32> using ftz32_simd = typename V::template rebind<F>;
}

export namespace native {
  template<bool Hardware> struct mask_traits<::ftz::basic_ftz32<Hardware>> { using type = bool; };
  template <bool Hardware> struct simd_traits<::ftz::basic_ftz32<Hardware>> { using storage_type = float; };
  template <bool Hardware, class V, class Self>
  struct simd_customization<::ftz::basic_ftz32<Hardware>,V,Self> {
    using simd = Self;
    using ftz32 = ::ftz::basic_ftz32<Hardware>;
    static constexpr bool hardware = Hardware;
    static constexpr std::size_t N = V::lanes;
    using register_type = V;
    using value_type = ftz32;
    using native_type = typename V::native_type;
    using bits_type = ::ftz::detail::ftz32_words<V>;
    using mask_type = typename V::mask_type;
    using mask = mask_type;
    using vector_mask_type = typename V::vector_mask_type;
    static constexpr std::size_t lanes = V::lanes;
    native_inline constexpr simd_customization() noexcept : value_(0.0f) {}
    native_inline constexpr simd_customization(float value) noexcept : simd_customization(V(value)) {}
    native_inline constexpr simd_customization(ftz32 value) noexcept : value_(value.to_float()) {}
    native_inline constexpr simd_customization(V value) noexcept
      : value_(import(value)) {}

    [[nodiscard]] native_inline constexpr operator V() const noexcept { return value_; }
    native_inline constexpr simd_customization(native_type value) noexcept requires (N>1) : simd_customization(V(value)) {}
    [[nodiscard]] native_inline constexpr operator native_type() const noexcept { return value_.value; }
    template <class... X> requires (sizeof...(X)==N && N>1) && (std::convertible_to<X,ftz32> && ...)
    native_inline constexpr simd_customization(X... x) noexcept((noexcept(static_cast<float>(x)) && ...))
      : simd_customization(V(static_cast<float>(x)...)) {}

    template <class T> requires (std::same_as<T,float> || std::same_as<T,ftz32>)
    native_inline constexpr simd_customization(std::array<T,N> const & values) noexcept : value_(load_memory(values.data()).to_native()) {}

    [[nodiscard]] native_inline constexpr native_pure V to_native() const noexcept { return value_; }
    [[nodiscard]] native_inline constexpr native_pure bits_type to_bits() const noexcept { return bits(); }
    [[nodiscard]] static native_inline constexpr native_const simd from_float(float value) noexcept { return value; }
    [[nodiscard]] static native_inline constexpr native_const simd from_native(V value) noexcept { return value; }
    // Caller promises normal, signed zero, infinity or any NaN lanes.
    // No classification or normalization; this is an explicit invariant escape.
    [[nodiscard]] static native_inline constexpr native_const simd unsafe_from_float32(V value) noexcept {
      return {canonical{}, value};
    }

    [[nodiscard]] static native_inline constexpr native_const simd unsafe_from_float32(float value) noexcept {
      return {canonical{}, V(value)};
    }

    [[nodiscard]] native_inline constexpr native_pure bits_type bits() const noexcept { return bridge::encode(value_); }
    [[nodiscard]] static native_inline constexpr native_const simd from_bits(bits_type value) noexcept {
      return simd(bridge::decode(value));
    }

    [[nodiscard]] static native_inline constexpr native_const simd from_bits(std::uint32_t value) noexcept {
      return from_bits(bits_type(value));
    }

    template <std::size_t Alignment = 1, class T> requires (std::same_as<T,float> || std::same_as<T,ftz32>)
    [[nodiscard]] static native_inline constexpr simd load_memory(T const * p) noexcept {
      if constexpr (std::same_as<T,float>) return simd(V::template load_memory<Alignment>(p));
      else {
        std::array<std::uint32_t,N> words;
        if consteval {
          for(std::size_t lane=0;lane<N;++lane) words[lane]=p[lane].to_bits();
        } else { std::memcpy(words.data(),p,sizeof(words)); }
        return unsafe_from_float32(V::from_bits(bits_type::load(words.data())));
      }
    }

    template <std::size_t Alignment = 1, class T> requires (std::same_as<T,float> || std::same_as<T,ftz32>)
    native_inline constexpr void store_memory(T * p) const noexcept {
      if constexpr (std::same_as<T,float>) value_.template store_memory<Alignment>(p);
      else {
        std::array<std::uint32_t,N> words; bits().store(words.data());
        if consteval {
          for(std::size_t lane=0;lane<N;++lane)
            p[lane]=ftz32::unsafe_from_float32(std::bit_cast<float>(words[lane]));
        } else { std::memcpy(p,words.data(),sizeof(words)); }
      }
    }
    // Legacy pointer spellings forward to the safe unaligned memory helpers.
    [[nodiscard]] static native_inline constexpr native_pure simd load(float const * p) noexcept {
      return load_memory(p);
    }

    [[nodiscard]] static native_inline constexpr native_pure simd loadu(float const * p) noexcept { return load_memory(p); }
    native_inline constexpr void store(float * p) const noexcept { store_memory(p); }
    native_inline constexpr void storeu(float * p) const noexcept { store_memory(p); }
    [[nodiscard]] static native_inline constexpr native_pure simd load_partial(float const * p, std::size_t n, float fill = 0) noexcept {
      std::array<float, lanes> temporary; temporary.fill(fill);
      for (std::size_t i=0;i<n;++i) temporary[i]=p[i];
      return loadu(temporary.data());
    }

    native_inline constexpr void store_partial(float * p, std::size_t n) const noexcept {
      std::array<float, lanes> temporary; storeu(temporary.data());
      for (std::size_t i=0;i<n;++i) p[i]=temporary[i];
    }

    [[nodiscard]] static native_inline constexpr native_pure simd load_bits(std::uint32_t const * p) noexcept {
      return from_bits(bits_type::load(p));
    }

    native_inline constexpr void store_bits(std::uint32_t * p) const noexcept { bits().store(p); }
    [[nodiscard]] static native_inline constexpr native_pure simd load_bits_partial(
        std::uint32_t const * p, std::size_t n, std::uint32_t fill = 0) noexcept {
      return from_bits(bits_type::load_partial(p, n, fill));
    }

    native_inline constexpr void store_bits_partial(std::uint32_t * p, std::size_t n) const noexcept {
      bits().store_partial(p, n);
    }

    [[nodiscard]] friend native_inline constexpr native_const simd operator+(simd a, simd b) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_add<Hardware>(a.value_, b.value_)};
    }

    [[nodiscard]] friend native_inline constexpr native_const simd operator-(simd a, simd b) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_sub<Hardware>(a.value_, b.value_)};
    }

    [[nodiscard]] friend native_inline constexpr native_const simd operator*(simd a, simd b) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_mul(a.value_, b.value_)};
    }

    [[nodiscard]] friend native_inline constexpr native_const simd operator/(simd a, simd b) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_div<Hardware>(a.value_, b.value_)};
    }

    native_inline constexpr simd & operator+=(simd b) noexcept { return static_cast<Self &>(*this) = static_cast<Self &>(*this) + b; }
    native_inline constexpr simd & operator-=(simd b) noexcept { return static_cast<Self &>(*this) = static_cast<Self &>(*this) - b; }
    native_inline constexpr simd & operator*=(simd b) noexcept { return static_cast<Self &>(*this) = static_cast<Self &>(*this) * b; }
    native_inline constexpr simd & operator/=(simd b) noexcept { return static_cast<Self &>(*this) = static_cast<Self &>(*this) / b; }
    [[nodiscard]] friend native_inline constexpr native_const simd operator-(simd a) noexcept {
      return {canonical{}, bridge::decode(a.bits() ^ bits_type(0x80000000u))};
    }

    [[nodiscard]] friend native_inline constexpr native_const simd operator+(simd a) noexcept { return a; }
    [[nodiscard]] friend native_inline constexpr native_const simd abs(simd a) noexcept {
      return {canonical{}, bridge::decode(a.bits() & bits_type(0x7fffffffu))};
    }

    [[nodiscard]] friend native_inline constexpr native_const mask_type operator<(simd a, simd b) noexcept { return a.value_ < b.value_; }
    [[nodiscard]] friend native_inline constexpr native_const mask_type operator>(simd a, simd b) noexcept { return a.value_ > b.value_; }
    [[nodiscard]] friend native_inline constexpr native_const mask_type operator==(simd a, simd b) noexcept { return a.value_ == b.value_; }
    [[nodiscard]] friend native_inline constexpr native_const mask_type operator!=(simd a, simd b) noexcept { return ~(a == b); }
    [[nodiscard]] friend native_inline constexpr native_const mask_type operator<=(simd a, simd b) noexcept { return (a < b) | (a == b); }
    [[nodiscard]] friend native_inline constexpr native_const mask_type operator>=(simd a, simd b) noexcept { return (a > b) | (a == b); }
    template<class M> requires (std::same_as<M,mask_type> || std::same_as<M,vector_mask_type>)
    [[nodiscard]] friend native_inline constexpr native_const simd select(M mask,simd a,simd b) noexcept {
      // Both mask domains select whole lanes, preserving chosen NaN payloads.
      return {canonical{},select(mask,a.value_,b.value_)};
    }

    template <class M, class A, class B, class E>
      requires (std::same_as<M,mask_type> || std::same_as<M,vector_mask_type>) &&
        (std::same_as<A,simd> || std::same_as<B,simd>) &&
        std::convertible_to<A,simd> && std::convertible_to<B,simd> &&
        (std::same_as<E,simd> || std::convertible_to<E,V>)
    [[nodiscard]] friend native_inline constexpr simd masked_scaleb(
        M active, A prior, B value, E exponent)
        noexcept(noexcept(simd(prior)) && noexcept(simd(value)) &&
          noexcept(scaling_exponent(exponent))) {
      simd imported_prior(prior), imported_value(value);
      V shift = scaling_exponent(exponent);
      V result = ::ftz::detail::ftz32_vector_scaleb(imported_value.value_, shift);
      return {canonical{},select(active,result,imported_prior.value_)};
    }

    template <class M, class A, class E>
      requires (std::same_as<M,mask_type> || std::same_as<M,vector_mask_type>) &&
        std::same_as<A,simd> && (std::same_as<E,simd> || std::convertible_to<E,V>)
    [[nodiscard]] friend native_inline constexpr simd masked_scaleb_zero(
        M active, A value, E exponent) noexcept(noexcept(scaling_exponent(exponent))) {
      V shift = scaling_exponent(exponent);
      V result = ::ftz::detail::ftz32_vector_scaleb(value.value_, shift);
      return {canonical{},select(active,result,V(0.f))};
    }

    template <class A, class E> requires std::same_as<A,simd> &&
      (std::same_as<E,simd> || std::convertible_to<E,V>)
    [[nodiscard]] friend native_inline constexpr simd scaleb(A value, E exponent)
        noexcept(noexcept(scaling_exponent(exponent))) {
      V shift = scaling_exponent(exponent);
      return {canonical{},::ftz::detail::ftz32_vector_scaleb(value.value_, shift)};
    }
    // An FTZ exponent alone does not change a raw base's arithmetic contract.
    // Explicit forwarding also prevents competing implicit native conversions.
    template <class M, class A, class B>
      requires (std::same_as<M,mask_type> || std::same_as<M,vector_mask_type>) &&
        std::same_as<A,V> && std::same_as<B,V> &&
        requires(M mask, V raw) { masked_scaleb(mask,raw,raw,raw); }
    [[nodiscard]] friend native_inline constexpr native_const V masked_scaleb(
        M active, A prior, B value, simd exponent) noexcept {
      return masked_scaleb(active,prior,value,exponent.value_);
    }

    template <class M, class A>
      requires (std::same_as<M,mask_type> || std::same_as<M,vector_mask_type>) && std::same_as<A,V> &&
        requires(M mask, V raw) { masked_scaleb_zero(mask,raw,raw); }
    [[nodiscard]] friend native_inline constexpr native_const V masked_scaleb_zero(
        M active, A value, simd exponent) noexcept {
      return masked_scaleb_zero(active,value,exponent.value_);
    }

    template <class A> requires std::same_as<A,V> && requires(V raw) { scaleb(raw,raw); }
    [[nodiscard]] friend native_inline constexpr native_const V scaleb(A value, simd exponent) noexcept {
      return scaleb(value,exponent.value_);
    }

    [[nodiscard]] friend native_inline constexpr native_const simd sqrt(simd a) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_sqrt(a.value_)};
    }

    template <class A, class B> requires std::convertible_to<A, simd> && std::convertible_to<B, simd>
    [[nodiscard]] friend native_inline constexpr simd fma(simd a, A b, B c)
        noexcept(noexcept(simd(b)) && noexcept(simd(c))) {
      return fused(a, simd(b), simd(c));
    }

    template <class A, class B> requires (!std::same_as<A, simd>) && std::convertible_to<A, simd> && std::convertible_to<B, simd>
    [[nodiscard]] friend native_inline constexpr simd fma(A a, simd b, B c)
        noexcept(noexcept(simd(a)) && noexcept(simd(c))) {
      return fused(simd(a), b, simd(c));
    }

    template <class A, class B> requires (!std::same_as<A, simd>) && (!std::same_as<B, simd>) &&
        std::convertible_to<A, simd> && std::convertible_to<B, simd>
    [[nodiscard]] friend native_inline constexpr simd fma(A a, B b, simd c)
        noexcept(noexcept(simd(a)) && noexcept(simd(b))) {
      return fused(simd(a), simd(b), c);
    }
 
    // Mixed operands select this value contract instead of escaping via the
    // deliberately implicit native-register conversion. No runtime dispatch.
    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator+(simd a, T b)
        noexcept(noexcept(a + simd(b))) { return a + simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator+(T a, simd b)
        noexcept(noexcept(simd(a) + b)) { return simd(a) + b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator-(simd a, T b)
        noexcept(noexcept(a - simd(b))) { return a - simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator-(T a, simd b)
        noexcept(noexcept(simd(a) - b)) { return simd(a) - b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator*(simd a, T b)
        noexcept(noexcept(a * simd(b))) { return a * simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator*(T a, simd b)
        noexcept(noexcept(simd(a) * b)) { return simd(a) * b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator/(simd a, T b)
        noexcept(noexcept(a / simd(b))) { return a / simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr simd operator/(T a, simd b)
        noexcept(noexcept(simd(a) / b)) { return simd(a) / b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator<(simd a, T b)
        noexcept(noexcept(a < simd(b))) { return a < simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator<(T a, simd b)
        noexcept(noexcept(simd(a) < b)) { return simd(a) < b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator>(simd a, T b)
        noexcept(noexcept(a > simd(b))) { return a > simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator>(T a, simd b)
        noexcept(noexcept(simd(a) > b)) { return simd(a) > b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator==(simd a, T b)
        noexcept(noexcept(a == simd(b))) { return a == simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator==(T a, simd b)
        noexcept(noexcept(simd(a) == b)) { return simd(a) == b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator!=(simd a, T b)
        noexcept(noexcept(a != simd(b))) { return a != simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator!=(T a, simd b)
        noexcept(noexcept(simd(a) != b)) { return simd(a) != b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator<=(simd a, T b)
        noexcept(noexcept(a <= simd(b))) { return a <= simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator<=(T a, simd b)
        noexcept(noexcept(simd(a) <= b)) { return simd(a) <= b; }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator>=(simd a, T b)
        noexcept(noexcept(a >= simd(b))) { return a >= simd(b); }

    template <class T> requires (!std::same_as<T, simd>) && std::convertible_to<T, simd>
    [[nodiscard]] friend native_inline constexpr mask_type operator>=(T a, simd b)
        noexcept(noexcept(simd(a) >= b)) { return simd(a) >= b; }
 
  private:
    
    using bridge = ::ftz::detail::ftz32_bridge<V>;
    template <class E>
    [[nodiscard]] static native_inline constexpr V scaling_exponent(E exponent)
        noexcept(noexcept(V(exponent))) {
      if constexpr (std::same_as<E,simd>) return exponent.value_;
      else {
        V raw(exponent);
        if constexpr(Hardware) return raw;
        else return bridge::decode(::ftz::detail::ftz32_import_words(bridge::encode(raw)));
      }
    }
    [[nodiscard]] static native_inline constexpr native_const V import(V value) noexcept {
      return bridge::decode(::ftz::detail::ftz32_import_words(bridge::encode(value)));
 
    }
    struct canonical {};
    V value_;
    native_inline constexpr simd_customization(canonical, V value) noexcept : value_(value) {}
    [[nodiscard]] static native_inline constexpr native_const simd fused(simd a, simd b, simd c) noexcept {
      return {canonical{}, ::ftz::detail::ftz32_vector_fma(a.value_, b.value_, c.value_)};
    }
  };
}
 
export namespace ftz {
  // A scalar FTZ operand promotes a raw register even when no FTZ SIMD
  // argument exists for hidden-friend lookup. Never fall back to built-in float.
  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator+(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) + detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator+(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) + detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator-(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) - detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator-(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) - detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator*(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) * detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator*(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) * detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator/(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) / detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator/(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) / detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator<(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) < detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator<(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) < detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator>(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) > detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator>(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) > detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator==(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) == detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator==(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) == detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator!=(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) != detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator!=(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) != detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator<=(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) <= detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator<=(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) <= detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator>=(V a,basic_ftz32<Hardware> b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) >= detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }

  template <bool Hardware, detail::raw_register V>
  [[nodiscard]] native_inline constexpr native_const auto operator>=(basic_ftz32<Hardware> a,V b) noexcept {
    return detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) >= detail::ftz32_simd<V,basic_ftz32<Hardware>>(b);
  }
  // Compound assignment keeps the destination type, while the FTZ operand
  // still chooses the arithmetic contract before publication to raw storage.
  template <bool Hardware, detail::raw_register V>
  native_inline constexpr V & operator+=(V & a,basic_ftz32<Hardware> b) noexcept {
    a=(detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) + detail::ftz32_simd<V,basic_ftz32<Hardware>>(b)).to_native();
    return a;
  }

  template <detail::raw_register V, detail::ftz32_vector R> requires std::same_as<typename R::register_type,V>
  native_inline constexpr V & operator+=(V & a,R b) noexcept {
    a=(R(a) + R(b)).to_native();
    return a;
  }

  template <bool Hardware, detail::raw_register V>
  native_inline constexpr V & operator-=(V & a,basic_ftz32<Hardware> b) noexcept {
    a=(detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) - detail::ftz32_simd<V,basic_ftz32<Hardware>>(b)).to_native();
    return a;
  }

  template <detail::raw_register V, detail::ftz32_vector R> requires std::same_as<typename R::register_type,V>
  native_inline constexpr V & operator-=(V & a,R b) noexcept {
    a=(R(a) - R(b)).to_native();
    return a;
  }

  template <bool Hardware, detail::raw_register V>
  native_inline constexpr V & operator*=(V & a,basic_ftz32<Hardware> b) noexcept {
    a=(detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) * detail::ftz32_simd<V,basic_ftz32<Hardware>>(b)).to_native();
    return a;
  }

  template <detail::raw_register V, detail::ftz32_vector R> requires std::same_as<typename R::register_type,V>
  native_inline constexpr V & operator*=(V & a,R b) noexcept {
    a=(R(a) * R(b)).to_native();
    return a;
  }

  template <bool Hardware, detail::raw_register V>
  native_inline constexpr V & operator/=(V & a,basic_ftz32<Hardware> b) noexcept {
    a=(detail::ftz32_simd<V,basic_ftz32<Hardware>>(a) / detail::ftz32_simd<V,basic_ftz32<Hardware>>(b)).to_native();
    return a;
  }

  template <detail::raw_register V, detail::ftz32_vector R> requires std::same_as<typename R::register_type,V>
  native_inline constexpr V & operator/=(V & a,R b) noexcept {
    a=(R(a) / R(b)).to_native();
    return a;
  }
  namespace detail {
    template <class... T> struct raw_family { using type = void; };
    template <class T,class... Rest> struct raw_family<T,Rest...> {
      using type = std::conditional_t<raw_register<T>,T,typename raw_family<Rest...>::type>;
    };
    template <class... X> using raw_family_t = typename raw_family<X...>::type;
    template <class... T> struct scalar_family { using type = void; };
    template <class T,class... Rest> struct scalar_family<T,Rest...> {
      using type = std::conditional_t<ftz32_type<T>,T,typename scalar_family<Rest...>::type>;
    };
    template <class... X> using scalar_family_t = typename scalar_family<X...>::type;
    template <class... X> concept ftz32_raw_scalar_fma =
      (ftz32_type<std::remove_cvref_t<X>> || ...) &&
      raw_register<raw_family_t<X...>> &&
      (std::convertible_to<X &,ftz32_simd<raw_family_t<X...>,scalar_family_t<X...>>> && ...);
  }
  template <class A,class B,class C> requires detail::ftz32_raw_scalar_fma<A,B,C>
  [[nodiscard]] native_inline constexpr auto fma(A a,B b,C c)
      noexcept(std::is_nothrow_constructible_v<detail::ftz32_simd<detail::raw_family_t<A,B,C>,detail::scalar_family_t<A,B,C>>,A &> &&
        std::is_nothrow_constructible_v<detail::ftz32_simd<detail::raw_family_t<A,B,C>,detail::scalar_family_t<A,B,C>>,B &> &&
        std::is_nothrow_constructible_v<detail::ftz32_simd<detail::raw_family_t<A,B,C>,detail::scalar_family_t<A,B,C>>,C &>) {
    using result_type=detail::ftz32_simd<detail::raw_family_t<A,B,C>,detail::scalar_family_t<A,B,C>>;
    return fma(result_type(a),result_type(b),result_type(c));
  }
}
 
#if !defined(__cpp_structured_bindings) || __cpp_structured_bindings < 202411L
#error "FTZ array math requires C++26 structured-binding packs (Clang 21+ with -std=c++2c or clang-cl /std:c++latest)."
#endif
 
export namespace ftz {
  namespace detail {
    template <class R> struct ftz32_native_for;
    template <ftz32_vector R> struct ftz32_native_for<R> { using type = typename R::register_type; };
    template <bool H> struct ftz32_native_for<basic_ftz32<H>> { using type = ::ftz::detail::native::fp32x1; };
    template <class R> using ftz32_native = typename ftz32_native_for<R>::type;
    template <class R> concept ftz32_value = requires { typename ftz32_native_for<R>::type; };
    template <ftz32_value R> [[nodiscard]] native_inline constexpr native_const ftz32_native<R> ftz32_unwrap(R value) noexcept {
      if constexpr (ftz32_type<R>) return ::ftz::detail::native::fp32x1(value.to_float());
      else return value.to_native();
    }
    template <ftz32_value R> [[nodiscard]] native_inline constexpr native_const R ftz32_wrap(ftz32_native<R> value) noexcept {
      if constexpr (ftz32_type<R>) return R::unsafe_from_float32(value.value);
      else return R::unsafe_from_float32(value);
    }
 
  }
 
  namespace detail::ftz32_math {
    template<auto Operation, detail::ftz32_value R, class... X>
    [[nodiscard]] constexpr R constant(R input, X... rest) noexcept {
      if constexpr(ftz32_type<R>)
        return R::unsafe_from_float32(std::bit_cast<float>(Operation(input.to_bits(),rest.to_bits()...)));
      else return R::unsafe_from_float32(detail::ftz32_constant<Operation>(input.to_native(),rest.to_native()...));
    }
    // The exceptional pair shares extraction and full-range reduction. Keep
    // bounded registers untouched and standalone sin/cos on their own graphs.
    template <bool Hardware, class V>
    [[nodiscard]] native_inline constexpr native_pure std::pair<V, V> repair_sincos(
        V sine, V cosine, detail::ftz32_words<V> mask, V original) noexcept {
      if (!detail::ftz32_any(mask)) return {sine, cosine};
      using B = detail::ftz32_bridge<V>; using U = detail::ftz32_words<V>;
      std::array<std::uint32_t, V::lanes> flags, input, sine_words, cosine_words;
      mask.store(flags.data()); B::encode(original).store(input.data());
      B::encode(sine).store(sine_words.data()); B::encode(cosine).store(cosine_words.data());
      for (std::size_t lane = 0; lane < V::lanes; ++lane) {
        if (flags[lane] != 0) {
          auto pair = detail::ftz32_sincos<Hardware>(input[lane]);
          sine_words[lane] = pair.sine; cosine_words[lane] = pair.cosine;
        }
      }
      return {B::decode(U::load(sine_words.data())), B::decode(U::load(cosine_words.data()))};
    }
    // All register chains enter the existing stage-interleaved polynomial in one
    // call. Only out-of-domain lanes use the scalar full-range/special-value path.
    template <detail::ftz32_value R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure auto sincos(std::array<R, N> const & input) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> sine{},cosine{};
        constexpr auto sine_word=[](std::uint32_t word) { return detail::ftz32_sincos<R::hardware>(word).sine; };
        constexpr auto cosine_word=[](std::uint32_t word) { return detail::ftz32_sincos<R::hardware>(word).cosine; };
        for(std::size_t i=0;i<N;++i) {
          sine[i]=constant<sine_word>(input[i]);
          cosine[i]=constant<cosine_word>(input[i]);
        }
        return std::pair{sine,cosine};
      }
      if constexpr (N == 0) return std::pair{std::array<R,0>{}, std::array<R,0>{}};
      else {
        using V = detail::ftz32_native<R>; using B = detail::ftz32_bridge<V>; using U = detail::ftz32_words<V>;
        auto const & [...input_register] = input;
        auto const [...original] = std::array{detail::ftz32_unwrap(input_register)...};
        auto bounded = [](V value) {
          return mask_bits<std::uint32_t>(U(0x46000000u) > (B::encode(value) & U(0x7fffffffu)));
        };
        auto const [...allowed] = std::array{bounded(original)...};
        auto [sine_values, cosine_values] = ::ftz::detail::native::sincos_ftz<R::hardware>(
          std::array{B::decode(B::encode(original) & allowed)...});
        auto const & [...sine] = sine_values;
        auto const & [...cosine] = cosine_values;
        auto const [...repaired] = std::array{repair_sincos<R::hardware>(
          sine, cosine, allowed ^ U(0xffffffffu), original)...};
        return std::pair{std::array{detail::ftz32_wrap<R>(repaired.first)...},
          std::array{detail::ftz32_wrap<R>(repaired.second)...}};
      }
    }
    template <bool Cosine, detail::ftz32_value R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R, N> trig_single(std::array<R, N> const & input) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> result{};
        for(std::size_t i=0;i<N;++i) {
          if constexpr(Cosine) result[i]=constant<detail::ftz32_cos<R::hardware>>(input[i]);
          else result[i]=constant<detail::ftz32_sin<R::hardware>>(input[i]);
        }
        return result;
      }
      if constexpr (N == 0) return {};
      else {
        using V = detail::ftz32_native<R>; using B = detail::ftz32_bridge<V>; using U = detail::ftz32_words<V>;
        auto const & [...input_register] = input;
        auto const [...original] = std::array{detail::ftz32_unwrap(input_register)...};
        auto const [...allowed] = std::array{mask_bits<std::uint32_t>(
            U(0x46000000u) > (B::encode(original) & U(0x7fffffffu)))...};
        auto const safe = std::array{B::decode(B::encode(original) & allowed)...};
        auto const [...value] = [&] {
          if constexpr (Cosine) return native::cos_ftz<R::hardware>(safe);
          else return native::sin_ftz<R::hardware>(safe);
        }();
        if constexpr (Cosine)
          return std::array{detail::ftz32_wrap<R>(detail::ftz32_repair<detail::ftz32_cos<R::hardware>>(
            value, allowed ^ U(0xffffffffu), original))...};
        else
          return std::array{detail::ftz32_wrap<R>(detail::ftz32_repair<detail::ftz32_sin<R::hardware>>(
            value, allowed ^ U(0xffffffffu), original))...};
      }
    }
    template <detail::ftz32_value R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R, N> sin(std::array<R, N> const & input) noexcept { return trig_single<false>(input); }
    template <detail::ftz32_value R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R, N> cos(std::array<R, N> const & input) noexcept { return trig_single<true>(input); }
    template <unsigned int Degree = 6, detail::ftz32_value R, std::size_t N> requires (Degree >= 1 && Degree <= 7)
    [[nodiscard]] native_inline constexpr native_pure std::array<R, N> exp(std::array<R, N> const & input) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> result{};
        for(std::size_t i=0;i<N;++i) result[i]=constant<detail::ftz32_exp<Degree>>(input[i]);
        return result;
      }
      if constexpr (N == 0) return {};
      else {
        auto const & [...input_register] = input;
        auto const [...value] = native::exp_ftz<Degree>(
          std::array{detail::ftz32_unwrap(input_register)...});
        return std::array{detail::ftz32_wrap<R>(value)...};
      }
    }
    template <detail::ftz32_value R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R, N> expm1(std::array<R, N> const & input) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> result{};
        for(std::size_t i=0;i<N;++i) result[i]=constant<detail::ftz32_expm1<R::hardware>>(input[i]);
        return result;
      }
      if constexpr (N == 0) return {};
      else {
        using U = detail::ftz32_words<detail::ftz32_native<R>>;
        auto const & [...input_register] = input;
        auto original = std::array{detail::ftz32_unwrap(input_register)...};
        auto [values, validity] = ::ftz::detail::native::expm1_checked<R::hardware>(original);
        auto const & [...x] = original;
        auto const & [...value] = values;
        auto const & [...valid] = validity;
        // The native graph classifies and safely evaluates once. Its validity is
        // 0/1 per lane; subtraction maps invalid lanes to the full repair mask.
        return std::array{detail::ftz32_wrap<R>(detail::ftz32_repair<detail::ftz32_expm1<R::hardware>>(
          value, valid - U(1), x))...};
      }
    }
  }
  template <detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr native_pure auto sincos(R input) noexcept {
    auto [sine_values, cosine_values] = detail::ftz32_math::sincos(std::array{input});
    auto [sine] = sine_values;
    auto [cosine] = cosine_values;
    return std::pair{sine, cosine};
  }

  template <detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr native_pure R sin(R input) noexcept { return detail::ftz32_math::sin(std::array{input})[0]; }
  template <detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr native_pure R cos(R input) noexcept { return detail::ftz32_math::cos(std::array{input})[0]; }
  template <unsigned int Degree = 6, detail::ftz32_vector R> requires (Degree >= 1 && Degree <= 7)
  [[nodiscard]] native_inline constexpr native_pure R exp(R input) noexcept {
    auto [value] = detail::ftz32_math::exp<Degree>(std::array{input});
    return value;
  }

  template <detail::ftz32_vector R, bool Flush, unsigned int Degree> requires (!Flush && Degree >= 1 && Degree <= 7)
  [[nodiscard]] native_inline constexpr native_pure R exp(R input, std::bool_constant<Flush>, std::integral_constant<unsigned int, Degree>) noexcept { return exp<Degree>(input); }
  template <detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr native_pure R expm1(R input) noexcept {
    auto [value] = detail::ftz32_math::expm1(std::array{input});
    return value;
  }
 
  namespace detail {
    template <char Op, class V> [[nodiscard]] native_inline constexpr native_const V ftz32_native_binary(V a,V b) noexcept {
      if constexpr(Op=='+') return a+b;
      else if constexpr(Op=='-') return a-b;
      else return a*b;
    }
    template <char Op, class R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R,N> ftz32_array_binary(
        std::array<R,N> const & a, std::array<R,N> const & b) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> result{};
        for(std::size_t i=0;i<N;++i) {
          if constexpr(Op=='+') result[i]=a[i]+b[i];
          else if constexpr(Op=='-') result[i]=a[i]-b[i];
          else result[i]=a[i]*b[i];
        }
        return result;
      }
      if constexpr (N == 0) return {};
      else {
        auto const & [...x] = a;
        auto const & [...y] = b;
        // Issue every native operation before classifying or repairing lanes.
        auto const [...native] = std::array{ftz32_native_binary<Op>(x.to_native(), y.to_native())...};
        if constexpr (R::hardware && (Op == '+' || Op == '-'))
          return std::array{R::unsafe_from_float32(native)...};
        auto const [...mask] = std::array{ftz32_repair_mask(native)...};
        constexpr auto repair = [] {
          if constexpr (Op == '+') return ftz32_add<R::hardware>;
          else if constexpr (Op == '-') return ftz32_sub<R::hardware>;
          else return ftz32_mul;
        }();
        return std::array{R::unsafe_from_float32(ftz32_repair<repair>(
          native, mask, x.to_native(), y.to_native()))...};
      }
    }
    template <class R, std::size_t N>
    [[nodiscard]] native_inline constexpr native_pure std::array<R,N> ftz32_array_fma(std::array<R,N> const & a,
        std::array<R,N> const & b, std::array<R,N> const & c) noexcept {
      if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
        std::array<R,N> result{};
        for(std::size_t i=0;i<N;++i) result[i]=fma(a[i],b[i],c[i]);
        return result;
      }
      if constexpr (N == 0) return {};
      else {
        auto const & [...x] = a;
        auto const & [...y] = b;
        auto const & [...z] = c;
        auto const [...native] = std::array{fma(x.to_native(), y.to_native(), z.to_native())...};
        auto const [...mask] = std::array{ftz32_repair_mask(native)...};
        return std::array{R::unsafe_from_float32(ftz32_repair<ftz32_fma>(
          native, mask, x.to_native(), y.to_native(), z.to_native()))...};
      }
    }
  }
  namespace detail {
    template<class T, std::size_t N> native_inline constexpr std::array<T,N> array_broadcast(T const & value) noexcept {
      std::array<T,N> result;
      auto & [...element] = result;
      ((element = value), ...);
      return result;
    }
  }
  template<class R, std::size_t N> requires detail::ftz32_vector<R>
  [[nodiscard]] native_inline constexpr std::array<R,N> add(std::array<R,N> const & a, std::array<R,N> const & b) noexcept {
    return detail::ftz32_array_binary<'+'>(a,b);
  }

  template<class R, std::size_t N, class B> requires detail::ftz32_vector<R> && std::convertible_to<B const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> add(std::array<R,N> const & a, B const & b)
      noexcept(std::is_nothrow_constructible_v<R,B const &>) {
    return add(a, detail::array_broadcast<R,N>(R(b)));
  }

  template<class R, std::size_t N, class A> requires detail::ftz32_vector<R> && std::convertible_to<A const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> add(A const & a, std::array<R,N> const & b)
      noexcept(std::is_nothrow_constructible_v<R,A const &>) {
    return add(detail::array_broadcast<R,N>(R(a)), b);
  }

  template<class R, std::size_t N> requires detail::ftz32_vector<R>
  [[nodiscard]] native_inline constexpr std::array<R,N> sub(std::array<R,N> const & a, std::array<R,N> const & b) noexcept {
    return detail::ftz32_array_binary<'-'>(a,b);
  }

  template<class R, std::size_t N, class B> requires detail::ftz32_vector<R> && std::convertible_to<B const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> sub(std::array<R,N> const & a, B const & b)
      noexcept(std::is_nothrow_constructible_v<R,B const &>) {
    return sub(a, detail::array_broadcast<R,N>(R(b)));
  }

  template<class R, std::size_t N, class A> requires detail::ftz32_vector<R> && std::convertible_to<A const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> sub(A const & a, std::array<R,N> const & b)
      noexcept(std::is_nothrow_constructible_v<R,A const &>) {
    return sub(detail::array_broadcast<R,N>(R(a)), b);
  }

  template<class R, std::size_t N> requires detail::ftz32_vector<R>
  [[nodiscard]] native_inline constexpr std::array<R,N> mul(std::array<R,N> const & a, std::array<R,N> const & b) noexcept {
    return detail::ftz32_array_binary<'*'>(a,b);
  }

  template<class R, std::size_t N, class B> requires detail::ftz32_vector<R> && std::convertible_to<B const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> mul(std::array<R,N> const & a, B const & b)
      noexcept(std::is_nothrow_constructible_v<R,B const &>) {
    return mul(a, detail::array_broadcast<R,N>(R(b)));
  }

  template<class R, std::size_t N, class A> requires detail::ftz32_vector<R> && std::convertible_to<A const &,R>
  [[nodiscard]] native_inline constexpr std::array<R,N> mul(A const & a, std::array<R,N> const & b)
      noexcept(std::is_nothrow_constructible_v<R,A const &>) {
    return mul(detail::array_broadcast<R,N>(R(a)), b);
  }

  template<class R, std::size_t N> requires detail::ftz32_vector<R>
  [[nodiscard]] native_inline constexpr std::array<R,N> fma(std::array<R,N> const & a,
      std::array<R,N> const & b, std::array<R,N> const & c) noexcept {
    return detail::ftz32_array_fma(a,b,c);
  }

  template<detail::ftz32_value R, std::size_t N>
  [[nodiscard]] native_inline constexpr auto sin(std::array<R,N> const & input) noexcept {
    return detail::ftz32_math::sin(input);
  }

  template<detail::ftz32_value R, std::size_t N>
  [[nodiscard]] native_inline constexpr auto cos(std::array<R,N> const & input) noexcept {
    return detail::ftz32_math::cos(input);
  }

  template<unsigned int Degree = 6, detail::ftz32_value R, std::size_t N> requires (Degree >= 1 && Degree <= 7)
  [[nodiscard]] native_inline constexpr auto exp(std::array<R,N> const & input) noexcept {
    return detail::ftz32_math::exp<Degree>(input);
  }

  template<detail::ftz32_value R, std::size_t N, bool Flush, unsigned int Degree> requires (!Flush && Degree >= 1 && Degree <= 7)
  [[nodiscard]] native_inline constexpr auto exp(std::array<R,N> const & input, std::bool_constant<Flush>, std::integral_constant<unsigned int, Degree>) noexcept { return exp<Degree>(input); }
  template<detail::ftz32_value R, std::size_t N>
  [[nodiscard]] native_inline constexpr auto expm1(std::array<R,N> const & input) noexcept {
    return detail::ftz32_math::expm1(input);
  }

  template<detail::ftz32_value R, std::size_t N>
  [[nodiscard]] native_inline constexpr auto sincos(std::array<R,N> const & input) noexcept {
    return detail::ftz32_math::sincos(input);
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> tanh(std::array<R,N> const & input) noexcept {
    if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
      std::array<R,N> result{};
      for(std::size_t i=0;i<N;++i)
        result[i]=detail::ftz32_math::constant<detail::ftz32_tanh<R::hardware>>(input[i]);
      return result;
    }
    if constexpr(N==0) return {};
    else {
      auto const & [...value]=input;
      auto const [...result]=detail::native::tanh_ftz<R::hardware>(
        std::array{detail::ftz32_unwrap(value)...});
      return {{detail::ftz32_wrap<R>(result)...}};
    }
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R tanh(R input) noexcept {
    auto [result]=::ftz::tanh(std::array{input});
    return result;
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> log(std::array<R,N> const & input) noexcept {
    if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
      std::array<R,N> result{};
      for(std::size_t i=0;i<N;++i)
        result[i]=detail::ftz32_math::constant<detail::ftz32_log<R::hardware>>(input[i]);
      return result;
    }
    if constexpr(N==0) return {};
    else {
      auto const & [...value]=input;
      auto const [...result]=detail::native::log_ftz<false,R::hardware>(
        std::array{detail::ftz32_unwrap(value)...});
      return {{detail::ftz32_wrap<R>(result)...}};
    }
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R log(R input) noexcept {
    auto [result]=::ftz::log(std::array{input});
    return result;
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> log1p(std::array<R,N> const & input) noexcept {
    if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
      std::array<R,N> result{};
      for(std::size_t i=0;i<N;++i)
        result[i]=detail::ftz32_math::constant<detail::ftz32_log1p<R::hardware>>(input[i]);
      return result;
    }
    if constexpr(N==0) return {};
    else {
      auto const & [...value]=input;
      auto const [...result]=detail::native::log_ftz<true,R::hardware>(
        std::array{detail::ftz32_unwrap(value)...});
      return {{detail::ftz32_wrap<R>(result)...}};
    }
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R log1p(R input) noexcept {
    auto [result]=::ftz::log1p(std::array{input});
    return result;
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> atan2(
      std::array<R,N> const & y,std::array<R,N> const & x) noexcept {
    if (std::is_constant_evaluated() || ::ftz::detail::ftz32_emulated<R>) {
      std::array<R,N> result{};
      for(std::size_t i=0;i<N;++i)
        result[i]=detail::ftz32_math::constant<detail::ftz32_atan2<R::hardware>>(y[i],x[i]);
      return result;
    }
    if constexpr(N==0) return {};
    else {
      auto const & [...a]=y;
      auto const & [...b]=x;
      auto const [...result]=detail::native::atan2_ftz<R::hardware>(
        std::array{detail::ftz32_unwrap(a)...},std::array{detail::ftz32_unwrap(b)...});
      return {{detail::ftz32_wrap<R>(result)...}};
    }
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R atan2(R y,R x) noexcept {
    auto [result]=::ftz::atan2(std::array{y},std::array{x});
    return result;
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr typename R::mask isnan(R value) noexcept {
    using U=typename R::bits_type;
    return (value.to_bits() & U(0x7fffffffu)) > U(0x7f800000u);
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr typename R::mask isinf(R value) noexcept {
    using U=typename R::bits_type;
    return (value.to_bits() & U(0x7fffffffu)) == U(0x7f800000u);
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr typename R::mask isfinite(R value) noexcept {
    using U=typename R::bits_type;
    return (value.to_bits() & U(0x7fffffffu)) < U(0x7f800000u);
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr typename R::mask signbit(R value) noexcept {
    using U=typename R::bits_type;
    return (value.to_bits() & U(0x80000000u)) != U(0);
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R copysign(R magnitude,R sign) noexcept {
    using U=typename R::bits_type;
    using V=typename R::register_type;
    return R::unsafe_from_float32(V::from_bits(
      (magnitude.to_bits() & U(0x7fffffffu)) | (sign.to_bits() & U(0x80000000u))));
  }
 
  // Integral-valued results cannot be subnormal; preserve the raw rounding result.
  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R floor(R value) noexcept {
    using ::native::floor;
    return R::unsafe_from_float32(floor(value.to_native()));
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> floor(std::array<R,N> const & input) noexcept {
    auto const & [...value]=input;
    return {{floor(value)...}};
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R ceil(R value) noexcept {
    using ::native::ceil;
    return R::unsafe_from_float32(ceil(value.to_native()));
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> ceil(std::array<R,N> const & input) noexcept {
    auto const & [...value]=input;
    return {{ceil(value)...}};
  }

  template<detail::ftz32_vector R>
  [[nodiscard]] native_inline constexpr R trunc(R value) noexcept {
    using ::native::trunc;
    return R::unsafe_from_float32(trunc(value.to_native()));
  }

  template<detail::ftz32_value R,std::size_t N>
  [[nodiscard]] native_inline constexpr std::array<R,N> trunc(std::array<R,N> const & input) noexcept {
    auto const & [...value]=input;
    return {{trunc(value)...}};
  }
}