ftz 0.0.1
Fast, reproducible floating-point arithmetic
Loading...
Searching...
No Matches
math.h
Go to the documentation of this file.
1// SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
2// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
10#pragma once
11#include <native/attributes.h>
12#include <ftz/config.h>
13#include <ftz/math/exp_coefficients.h>
14#include <array>
15#include <bit>
16#include <utility>
17#include <cstdint>
18#if defined(__x86_64__) || defined(_M_X64)
19#include <immintrin.h>
20#elif defined(__aarch64__) || defined(_M_ARM64)
21#include <arm_neon.h>
22#endif
23import native.scalar;
24import native.math;
25
26namespace ftz::detail::native {
27 using ::native::mask_bits;
28 using fp32x1 = ::native::simd<float,1,::native::scalar>;
29 template <class V> concept float_register = requires { typename V::bits_type; V::lanes; };
30 namespace detail {
31 template <class V> struct fp32_bit_bridge {
32 using bits_type = typename V::bits_type;
33 static constexpr native_inline bits_type encode(V value) noexcept { return value.bits(); }
34 static constexpr native_inline V decode(bits_type value) noexcept { return V::from_bits(value); }
35 };
36 template <class V> struct trig_conversion {
37 using U = typename V::bits_type;
38 using I = typename V::template rebind<std::int32_t>;
39 static constexpr native_inline U integer(V value) noexcept {
40 return U::from_native(__builtin_bit_cast(typename U::native_type, convert<std::int32_t>(value).to_native()));
41 }
42 static constexpr native_inline V floating(U value) noexcept {
43 return convert<float>(I::from_native(__builtin_bit_cast(typename I::native_type, value.to_native())));
44 }
45 };
46 template <class V> constexpr native_inline V max(V a,V b) noexcept { return select(a>b,a,b); }
47 }
48
49// FTZ owns one host/shader exponential graph. Its finite overflow endpoint
50// matches native::exp<true>; reconstruction alone is instruction-specific.
51 namespace detail {
52 template <float_register V>
53 constexpr native_inline V exp_factor(V biased) noexcept {
54 if consteval {
55 std::array<float, V::lanes> lanes{};
56 std::array<std::uint32_t, V::lanes> words{};
57 biased.store(lanes.data());
58 for (std::size_t i = 0; i < V::lanes; ++i) {
59 auto const bits = std::bit_cast<std::uint32_t>(lanes[i]);
60 if (bits <= 0x437e0000u) words[i] = std::uint32_t(lanes[i]) << 23;
61 }
62 return V::from_bits(V::bits_type::load(words.data()));
63 }
64#if defined(__aarch64__) || defined(_M_ARM64)
65 // FCVTZU defines NaN and negative results as zero. No NaN admission mask
66 // is needed: the polynomial's NaN survives the final multiplication.
67 return V::from_bits(::native::fcvtzu(biased).template left<23>());
68#elif defined(__x86_64__) || defined(_M_X64)
69 // CVTT's indefinite result is INT_MIN; PMAXSD maps it and negative
70 // exponent fields to zero. This private reconstruction is not scaleb.
71 if constexpr (V::lanes == 1) {
72 auto const index = _mm_cvttss_si32(_mm_set_ss(biased.to_native()));
73 return V::from_bits(std::uint32_t(index > 0 ? index : 0) << 23);
74 } else if constexpr (V::lanes == 2 || V::lanes == 3) {
75 return V::from_storage(exp_factor(biased.to_storage()));
76 } else if constexpr (V::lanes == 4) {
77 auto const index = _mm_max_epi32(_mm_cvttps_epi32(biased.to_native()), _mm_setzero_si128());
78 return V::from_native(_mm_castsi128_ps(_mm_slli_epi32(index, 23)));
79 } else if constexpr (V::lanes == 8) {
80 auto const index = _mm256_max_epi32(_mm256_cvttps_epi32(biased.to_native()), _mm256_setzero_si256());
81 return V::from_native(_mm256_castsi256_ps(_mm256_slli_epi32(index, 23)));
82 }
83#else
84 // The portable conversion keeps its finite, representable precondition.
85 auto const safe = select((biased >= V(0.f)) & (biased <= V(254.f)), biased, V(0.f));
86 return V::from_bits(trig_conversion<V>::integer(safe).template left<23>());
87#endif
88 }
89 }
90 template <unsigned int Degree = 6, float_register V, std::size_t N> requires (Degree >= 1 && Degree <= 7)
91 native_flatten constexpr native_inline std::array<V, N> exp_ftz(std::array<V, N> const & input) noexcept {
92 if constexpr (N == 0) return {};
93 else {
94 auto const & [...x] = input;
95 auto const [...active] = std::array{(!(x < V(-87.33654022216796875f)))...};
96 // Classify independently of the arithmetic chain. This is the last
97 // binary32 input whose exact exponential rounds finite. NaNs stay on the
98 // arithmetic path without a separate comparison or restoration select.
99 auto const [...overflow] = std::array{(x > V(88.72283172607421875f))...};
100 auto const [...in_range] = std::array{(active & !overflow)...};
101 auto const [...replacement] = std::array{select(overflow,
102 V(std::bit_cast<float>(0x7f800000u)), V(0.f))...};
103 auto [...r] = std::array{x...};
104 auto const [...n] = std::array{round_even(r * V(1.4426950408889634f))...};
105 ((r = fma(n, V(-0x1.62e400p-1f), r)), ...);
106 ((r = fma(n, V(-0x1.7f7d1cp-20f), r)), ...);
107 using C = ::ftz::detail::math::exp_coefficients<Degree>;
108 auto const coefficient = [](unsigned int word) noexcept { return V(std::bit_cast<float>(word)); };
109 auto [...y] = std::array{fma(r, coefficient(C::leading), coefficient(C::next))...};
110 if constexpr (Degree >= 7) ((y = fma(r, y, coefficient(C::c5))), ...);
111 if constexpr (Degree >= 6) ((y = fma(r, y, coefficient(C::c4))), ...);
112 if constexpr (Degree >= 5) ((y = fma(r, y, coefficient(C::c3))), ...);
113 if constexpr (Degree >= 4) ((y = fma(r, y, coefficient(C::c2))), ...);
114 if constexpr (Degree >= 3) ((y = fma(r, y, coefficient(C::c1))), ...);
115 if constexpr (Degree >= 2) ((y = fma(r, y, coefficient(C::c0))), ...);
116 if constexpr (requires { masked_scaleb_zero(active...[0], y...[0], n...[0]); }) {
117 return {{masked_scaleb(in_range, replacement, y, n)...}};
118 } else {
119 // Active exponents are [-126,128]. Only n=128 needs a second factor;
120 // its first product is exact and normal. Retain all lower products
121 // directly, including their behavior under mixed DAZ/FTZ controls.
122 auto const [...high] = std::array{(n > V(127.f))...};
123 auto const [...factor] = std::array{detail::exp_factor(n + select(high, V(126.f), V(127.f)))...};
124 ((y = y * factor), ...);
125 ((y = select(high, select(high, y, V(0.f)) * V(2.f), y)), ...);
126 return {{select(in_range, y, replacement)...}};
127 }
128 }
129 }
130 template <unsigned int Degree = 6, float_register V> requires (Degree >= 1 && Degree <= 7)
131 constexpr native_inline V exp_ftz(V input) noexcept { return exp_ftz<Degree>(std::array{input})[0]; }
132
133// Altered source: paired polynomial and reducer with signed input FTZ,
134// explicit tiny-input selection and bitwise reconstruction. Notices are retained below.
135// Require binary32 RNE, native fused fma and no implicit expression contraction.
136 enum class trig_output { sine, cosine, pair };
137 namespace detail {
138 template <trig_output Output, bool Hardware, bool Bounded, float_register V, std::size_t N>
139 native_flatten constexpr native_inline auto sincos_ftz_kernel(std::array<V, N> const & input) noexcept {
140 using B = fp32_bit_bridge<V>;
141 using I = typename B::bits_type;
142 using C = trig_conversion<V>;
143 if constexpr (N == 0) {
144 if constexpr (Output == trig_output::pair) return std::pair{std::array<V,0>{}, std::array<V,0>{}};
145 else return std::array<V,0>{};
146 } else {
147 // Bounded lanes can choose either polynomial according to quadrant.
148 constexpr bool need_sine = Bounded || Output != trig_output::cosine;
149 constexpr bool need_cosine = Bounded || Output != trig_output::sine;
150 auto const constant = [](std::uint32_t word) noexcept { return B::decode(I(word)); };
151 auto const & [...original] = input;
152 // Integer admission/masking is independent of DAZ and preserves -0.
153 auto const [...word] = std::array{B::encode(original)...};
154 auto const [...canonical] = std::array{B::decode(word &
155 ((mask_bits<std::uint32_t>((word & I(0x7f800000u)) == I(0)) &
156 I(0x007fffffu)) ^ I(0xffffffffu)))...};
157 auto const [...magnitude] = std::array{B::decode(B::encode(canonical) & I(0x7fffffffu))...};
158 auto [...index] = std::array<I, N>{};
159 auto [...reduced] = std::array{canonical...};
160 if constexpr (Bounded) {
161 auto const [...product] = std::array{(magnitude * constant(0x3fa2f983u))...};
162 ((index = (C::integer(product) + I(1)) & I(0xfffffffeu)), ...);
163 auto const [...multiple] = std::array{C::floating(index)...};
164 ((reduced = fma(multiple, constant(0xbf490000u), magnitude)), ...);
165 ((reduced = fma(multiple, constant(0xb97da000u), reduced)), ...);
166 ((reduced = fma(multiple, constant(0xb3222169u), reduced)), ...);
167 }
168 auto const [...active] = std::array{
169 mask_bits<std::uint32_t>((B::encode(reduced) & I(0x7fffffffu)) > I(0x39800000u))...};
170 auto const [...masked] = [&] {
171 if constexpr(Hardware) return std::array{reduced...};
172 else return std::array{B::decode(B::encode(reduced) & active)...};
173 }();
174 auto const [...square] = std::array{(masked * masked)...};
175 auto [...sine] = [&] {
176 if constexpr (need_sine) return std::array{fma(constant(0xb94ca1f9u), square, constant(0x3c08839eu))...};
177 else return std::array<V,0>{};
178 }();
179 auto [...cosine] = [&] {
180 if constexpr (need_cosine) return std::array{fma(constant(0x37ccf5ceu), square, constant(0xbab6061au))...};
181 else return std::array<V,0>{};
182 }();
183 if constexpr (need_sine) ((sine = fma(sine, square, constant(0xbe2aaaa3u))), ...);
184 if constexpr (need_cosine) ((cosine = fma(cosine, square, constant(0x3d2aaaa5u))), ...);
185 if constexpr (need_sine) ((sine = sine * square), ...);
186 if constexpr (need_cosine) ((cosine = cosine * square), ...);
187 if constexpr (need_cosine) ((cosine = cosine * square), ...);
188 if constexpr (need_cosine) {
189 auto const [...half_square] = std::array{(square * V(0.5f))...};
190 ((cosine = cosine - half_square), ...);
191 }
192 if constexpr (need_sine) ((sine = fma(sine, masked, reduced)), ...);
193 if constexpr (need_cosine) ((cosine = cosine + V(1.0f)), ...);
194 // The select is essential for a tiny negative zero: +0 + -0 is +0.
195 if constexpr (need_sine) ((sine = B::decode((B::encode(sine) & active) |
196 (B::encode(reduced) & (active ^ I(0xffffffffu))))), ...);
197 if constexpr (Bounded) {
198 auto const [...bounded_sine] = [&] {
199 if constexpr (Output != trig_output::cosine) return std::array{B::decode(select((index & I(2)) == I(0),
200 B::encode(sine), B::encode(cosine)) ^ (B::encode(canonical) & I(0x80000000u)) ^
201 ((index & I(4)).template left<29>()))...};
202 else return std::array<V,0>{};
203 }();
204 if constexpr (Output != trig_output::sine)
205 ((cosine = B::decode(select((index & I(2)) == I(0), B::encode(cosine), B::encode(sine)) ^
206 (((index - I(2)) ^ I(0xffffffffu)) & I(4)).template left<29>())), ...);
207 if constexpr (Output == trig_output::pair) return std::pair{std::array{bounded_sine...}, std::array{cosine...}};
208 else if constexpr (Output == trig_output::sine) return std::array{bounded_sine...};
209 else return std::array{cosine...};
210 } else {
211 if constexpr (Output == trig_output::pair) return std::pair{std::array{sine...}, std::array{cosine...}};
212 else if constexpr (Output == trig_output::sine) return std::array{sine...};
213 else return std::array{cosine...};
214 }
215 }
216 }
217 } // namespace detail
218
219 // Reduced entry: finite |x| <= 1. Input subnormals become signed zero.
220 // No range reduction. This is a reproducible approximation, not CR sin/cos.
221 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
222 constexpr native_inline std::pair<std::array<V, N>, std::array<V, N>> sincos_reduced_ftz(std::array<V, N> const & x) noexcept {
223 return detail::sincos_ftz_kernel<trig_output::pair,Hardware,false>(x);
224 }
225 // Bounded entry: finite |x| < 8192. Shared three-FMA reduction.
226 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
227 constexpr native_inline std::pair<std::array<V, N>, std::array<V, N>> sincos_ftz(std::array<V, N> const & x) noexcept {
228 return detail::sincos_ftz_kernel<trig_output::pair,Hardware,true>(x);
229 }
230 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
231 constexpr native_inline std::array<V, N> sin_reduced_ftz(std::array<V, N> const & x) noexcept {
232 return detail::sincos_ftz_kernel<trig_output::sine,Hardware,false>(x);
233 }
234 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
235 constexpr native_inline std::array<V, N> cos_reduced_ftz(std::array<V, N> const & x) noexcept {
236 return detail::sincos_ftz_kernel<trig_output::cosine,Hardware,false>(x);
237 }
238 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
239 constexpr native_inline std::array<V, N> sin_ftz(std::array<V, N> const & x) noexcept { return detail::sincos_ftz_kernel<trig_output::sine,Hardware,true>(x); }
240 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
241 constexpr native_inline std::array<V, N> cos_ftz(std::array<V, N> const & x) noexcept { return detail::sincos_ftz_kernel<trig_output::cosine,Hardware,true>(x); }
242 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline std::pair<V, V> sincos_reduced_ftz(V x) noexcept {
243 auto [sine, cosine] = sincos_reduced_ftz<Hardware>(std::array{x});
244 return {sine[0], cosine[0]};
245 }
246 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline std::pair<V, V> sincos_ftz(V x) noexcept {
247 auto [sine, cosine] = sincos_ftz<Hardware>(std::array{x});
248 return {sine[0], cosine[0]};
249 }
250 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline V sin_reduced_ftz(V x) noexcept { return sin_reduced_ftz<Hardware>(std::array{x})[0]; }
251 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline V cos_reduced_ftz(V x) noexcept { return cos_reduced_ftz<Hardware>(std::array{x})[0]; }
252 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline V sin_ftz(V x) noexcept { return sin_ftz<Hardware>(std::array{x})[0]; }
253 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline V cos_ftz(V x) noexcept { return cos_ftz<Hardware>(std::array{x})[0]; }
254 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
255 constexpr native_inline std::pair<float, float> sincos_reduced_ftz(float x) noexcept {
256 auto [sine, cosine] = sincos_reduced_ftz<Hardware>(fp32x1(x)); return {sine.value, cosine.value};
257 }
258 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
259 constexpr native_inline std::pair<float, float> sincos_ftz(float x) noexcept {
260 auto [sine, cosine] = sincos_ftz<Hardware>(fp32x1(x)); return {sine.value, cosine.value};
261 }
262 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
263 constexpr native_inline float sin_reduced_ftz(float x) noexcept { return sin_reduced_ftz<Hardware>(fp32x1(x)).value; }
264 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
265 constexpr native_inline float cos_reduced_ftz(float x) noexcept { return cos_reduced_ftz<Hardware>(fp32x1(x)).value; }
266 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
267 constexpr native_inline float sin_ftz(float x) noexcept { return sin_ftz<Hardware>(fp32x1(x)).value; }
268 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
269 constexpr native_inline float cos_ftz(float x) noexcept { return cos_ftz<Hardware>(fp32x1(x)).value; }
270} // namespace ftz::detail::native
271
272/*
273 AVX implementation of sin, cos, sincos, exp and log
274
275 Based on "sse_mathfun.h", by Julien Pommier
276 http://gruntthepeon.free.fr/ssemath/
277
278 Copyright (C) 2012 Giovanni Garberoglio
279 Interdisciplinary Laboratory for Computational Science (LISC)
280 Fondazione Bruno Kessler and University of Trento
281 via Sommarive, 18
282 I-38123 Trento (Italy)
283
284 This software is provided 'as-is', without any express or implied
285 warranty. In no event will the authors be held liable for any damages
286 arising from the use of this software.
287
288 Permission is granted to anyone to use this software for any purpose,
289 including commercial applications, and to alter it and redistribute it
290 freely, subject to the following restrictions:
291
292 1. The origin of this software must not be misrepresented; you must not
293 claim that you wrote the original software. If you use this software
294 in a product, an acknowledgment in the product documentation would be
295 appreciated but is not required.
296 2. Altered source versions must be plainly marked as such, and must not be
297 misrepresented as being the original software.
298 3. This notice may not be removed or altered from any source distribution.
299*/
300
301/* RTS repository license (retained verbatim):
302Software License Agreement (BSD 2-Clause License)
303========================================
304
305Copyright 2017 Edward Kmett
306
307Redistribution and use in source and binary forms, with or without
308modification, are permitted provided that the following conditions are met:
309
310 * Redistributions of source code must retain the above copyright
311 notice, this list of conditions and the following disclaimer.
312
313 * Redistributions in binary form must reproduce the above copyright
314 notice, this list of conditions and the following disclaimer in the
315 documentation and/or other materials provided with the distribution.
316
317THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
318ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
319WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
320DISCLAIMED. IN NO EVENT SHALL YAHOO! INC. BE LIABLE FOR ANY
321DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
322(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
323LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
324ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
325(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
326SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
327*/
328
329/*
330 * SPDX-FileCopyrightText: 2012 Giovanni Garberoglio
331 * SPDX-FileCopyrightText: 2017 Edward Kmett
332 * SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
333 * SPDX-License-Identifier: Zlib AND BSD-2-Clause
334 */
335
336namespace ftz::detail::native {
337 template <class F, class U> struct expm1_result {
338 F value;
339 U valid;
340 };
341 namespace detail {
342 template <bool Hardware, bool Gain, float_register V, std::size_t N>
343 native_flatten constexpr native_inline auto expm1_graph(std::array<V, N> const & input) noexcept {
344 using B = fp32_bit_bridge<V>;
345 using U = typename B::bits_type;
346 using result_type = expm1_result<std::array<V, N>, std::array<U, N>>;
347 if constexpr (N == 0) return result_type{};
348 else {
349 auto const constant = [](std::uint32_t word) noexcept { return B::decode(U(word)); };
350 auto const & [...original] = input;
351 auto [...word] = std::array{B::encode(original)...};
352 U const finite_limit(0x7f800000u), positive_limit(0x3f800001u), sign(0x80000000u);
353 auto [...valid] = std::array<U, N>{};
354 // Raw unsigned intervals admit the domain without evaluating even sNaNs.
355 if constexpr (Gain) {
356 // Nonnegative finite values, plus negative zero.
357 ((valid = mask_bits<std::uint32_t>((finite_limit > word) | (word == sign))), ...);
358 } else {
359 // Negative finite values, or nonnegative values through one inclusive.
360 ((valid = mask_bits<std::uint32_t>(
361 (finite_limit > (word ^ sign)) | (positive_limit > word))), ...);
362 }
363 ((word = word & valid), ...);
364 ((word = select((word & U(0x7f800000u)) == U(0), word & U(0x80000000u), word)), ...);
365 if constexpr (Gain) ((word = word ^ U(0x80000000u)), ...);
366 auto const [...x] = std::array{max(B::decode(word), V(-18))...};
367 auto const [...n] = std::array{round_even(x * constant(0x3fb8aa3bu))...};
368 auto [...r] = std::array{fma(n, constant(0xbf317200u), x)...};
369 ((r = fma(n, constant(0xb5bfbe8eu), r)), ...);
370 ((r = select(n == V(0), x, r)), ...);
371 // The tiny result is selected from r after speculative polynomial work.
372 auto const [...t] = [&] {
373 if constexpr(Hardware) return std::array{r...};
374 else return std::array{B::decode(select(
375 (B::encode(r) & U(0x7fffffffu)) > U(0x33000000u),
376 B::encode(r), U(0)))...};
377 }();
378 auto const [...z] = std::array{(t * t)...};
379 auto [...h] = std::array{fma(t, constant(0x3493f27eu), constant(0x3638ef1du))...};
380 ((h = fma(t, h, constant(0x37d00d01u))), ...);
381 ((h = fma(t, h, constant(0x39500d01u))), ...);
382 ((h = fma(t, h, constant(0x3ab60b61u))), ...);
383 ((h = fma(t, h, constant(0x3c088889u))), ...);
384 ((h = fma(t, h, constant(0x3d2aaaabu))), ...);
385 ((h = fma(t, h, constant(0x3e2aaaabu))), ...);
386 ((h = fma(t, h, constant(0x3f000000u))), ...);
387 auto [...p] = std::array{fma(z, h, r)...};
388 if constexpr(Hardware) {
389 ((p = B::decode(select(
390 (B::encode(r) & U(0x7fffffffu)) > U(0x33000000u),
391 B::encode(p), B::encode(r)))), ...);
392 } else {
393 ((p = select(t == V(0), r, p)), ...);
394 }
395 // 2^n-1 is exact for -24 <= n <= 1. Clamp speculative lanes before it.
396 auto const [...scale] = std::array{normal_pow2(max(n, V(-24)))...};
397 auto [...result] = std::array{fma(scale, p, scale - V(1))...};
398 ((result = select(n == V(0), p, result)), ...);
399 ((result = select(n == V(-25), select(p > V(0), constant(0xbf7fffffu), V(-1)), result)), ...);
400 ((result = select(n < V(-25), V(-1), result)), ...);
401 // The entry-canonicalized graph returns only normal or signed zero.
402 if constexpr(Hardware) {
403 ((word = B::encode(result)), ...);
404 } else {
405 ((word = B::encode(flush_to_zero(result))), ...);
406 }
407 if constexpr (Gain) ((word = word ^ U(0x80000000u)), ...);
408 ((word = word & valid), ...);
409 return result_type{{{B::decode(word)...}}, {{(valid & U(1))...}}};
410 }
411 }
412 }
413 // Conditional reproducible graph: actual RNE fused FMA, separate RNE multiply.
414 // Signed FTZ at entry/exit; no ambient FP-mode changes. Finite x <= 1 only.
415 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
416 constexpr native_inline auto expm1_checked(std::array<V, N> const & input) noexcept {
417 return detail::expm1_graph<Hardware,false>(input);
418 }
419 // Damping gain -expm1(-a), finite a >= 0 (either signed zero accepted).
420 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V, std::size_t N>
421 constexpr native_inline auto damping_gain_checked(std::array<V, N> const & input) noexcept {
422 return detail::expm1_graph<Hardware,true>(input);
423 }
424 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline auto expm1_checked(V input) noexcept {
425 auto r = expm1_checked<Hardware>(std::array{input});
426 return expm1_result<V, typename detail::fp32_bit_bridge<V>::bits_type>{
427 r.value[0], r.valid[0]};
428 }
429 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0, float_register V> constexpr native_inline auto damping_gain_checked(V input) noexcept {
430 auto r = damping_gain_checked<Hardware>(std::array{input});
431 return expm1_result<V, typename detail::fp32_bit_bridge<V>::bits_type>{
432 r.value[0], r.valid[0]};
433 }
434 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
435 constexpr native_inline expm1_result<float, std::uint32_t> expm1_checked(float input) noexcept {
436 auto r = expm1_checked<Hardware>(fp32x1(input));
437 return {r.value.value, r.valid.value};
438 }
439 template <bool Hardware = FTZ_FP32_HARDWARE_FTZ != 0>
440 constexpr native_inline expm1_result<float, std::uint32_t> damping_gain_checked(float input) noexcept {
441 auto r = damping_gain_checked<Hardware>(fp32x1(input));
442 return {r.value.value, r.valid.value};
443 }
444
445 // Same live Horner chains as the scalar tanh graph, selected per lane.
446 // Input comes from typed FTZ values: normal/signed zero/infinity/any NaN.
447 // This is a prechecked graph, not an entry point for raw subnormal inputs.
448 // For active outputs, |x| in (2^-12,10) gives normal z=x*x in
449 // [2^-24,100]. The seven
450 // affine transforms have |t|<=1; cancellation near their exact dyadic
451 // centers gives t=0 or |t|>=2^-24. Interval propagation with outward
452 // binary32 rounding bounds keeps every live Horner stage at |h|>2^-22
453 // and |h|<2. The final product is normal as well. Internal FTZ work is
454 // therefore unnecessary for either policy. Hardware tiny lanes may flush
455 // their unused square/product; their t stays in [-1,1] and their final
456 // output is the original word. Leading zero coefficients
457 // add only exact 0*t+0 / 0*t+c steps before each original live chain.
458 template <bool Hardware, float_register V, std::size_t N>
459 native_flatten constexpr native_inline std::array<V,N> tanh_ftz(std::array<V,N> const & input) noexcept {
460 using B=detail::fp32_bit_bridge<V>;
461 using U=typename B::bits_type;
462 if constexpr (N==0) return {};
463 else {
464 auto const & [...original]=input;
465 auto const [...word]=std::array{B::encode(original)...};
466 auto const [...magnitude]=std::array{(word & U(0x7fffffffu))...};
467 // Both paths bound large/nonfinite inputs before squaring. Manual FTZ
468 // also avoids subnormal intermediate work; hardware FTZ handles tiny
469 // lanes naturally, and the required tiny-region selection replaces them.
470 auto const [...safe]=[&] {
471 if constexpr(Hardware) return std::array{
472 select(magnitude<U(0x41200000u),magnitude,U(0x3f800000u))...};
473 else return std::array{select((magnitude>U(0x39800000u)) &
474 (magnitude<U(0x41200000u)),magnitude,U(0x3f800000u))...};
475 }();
476 auto const [...x]=std::array{B::decode(safe)...};
477 auto const [...z]=std::array{(x*x)...};
478 // Retain one interval index, not seven masks or eleven coefficient packs.
479 auto const [...interval]=std::array{select(safe<=U(0x40800000u),
480 select(safe<=U(0x40000000u),select(safe<=U(0x3f800000u),U(0),U(1)),
481 select(safe<=U(0x40400000u),U(2),U(3))),
482 select(safe<=U(0x41000000u),select(safe<=U(0x40c00000u),U(4),U(5)),U(6)))...};
483 auto coefficient=[](U index,unsigned c0,unsigned c1,unsigned c2,unsigned c3,
484 unsigned c4,unsigned c5,unsigned c6) noexcept {
485 if constexpr(U::lanes==1) {
486 std::array table{c0,c1,c2,c3,c4,c5,c6,c6};
487 return B::decode(U(table[index.to_native()]));
488 }
489#if defined(__x86_64__) || defined(_M_X64)
490 else if constexpr(U::lanes<=8) {
491 auto table=__builtin_bit_cast(__m256i, std::array{c0,c1,c2,c3,c4,c5,c6,c6});
492 if constexpr(U::lanes<=4) {
493 auto i=_mm256_zextsi128_si256(__builtin_bit_cast(__m128i, index.to_native()));
494 auto bits=_mm256_castsi256_si128(_mm256_permutevar8x32_epi32(table,i));
495 return B::decode(U::from_native(__builtin_bit_cast(typename U::native_type, bits)));
496 } else {
497 auto bits=_mm256_permutevar8x32_epi32(table,__builtin_bit_cast(__m256i, index.to_native()));
498 return B::decode(U::from_native(__builtin_bit_cast(typename U::native_type, bits)));
499 }
500 } else {
501 auto table=__builtin_bit_cast(__m512i, std::array{c0,c1,c2,c3,c4,c5,c6,c6,c0,c1,c2,c3,c4,c5,c6,c6});
502 auto bits=_mm512_permutexvar_epi32(__builtin_bit_cast(__m512i, index.to_native()),table);
503 return B::decode(U::from_native(__builtin_bit_cast(typename U::native_type, bits)));
504 }
505#elif defined(__aarch64__) || defined(_M_ARM64)
506 else {
507 uint8x16x2_t table{{__builtin_bit_cast(uint8x16_t, std::array{c0,c1,c2,c3}),
508 __builtin_bit_cast(uint8x16_t, std::array{c4,c5,c6,c6})}};
509 auto i=__builtin_bit_cast(uint32x4_t, index.to_native());
510 auto offsets=vaddq_u32(vmulq_n_u32(i,0x04040404u),vdupq_n_u32(0x03020100u));
511 auto bits=vqtbl2q_u8(table,vreinterpretq_u8_u32(offsets));
512 return B::decode(U::from_native(__builtin_bit_cast(typename U::native_type, bits)));
513 }
514#endif
515 };
516 auto const [...t]=std::array{fma(z,
517 coefficient(interval,0x40000000u,0x3f000000u,0x3e800000u,0x3e800000u,0x3d800000u,0x3d800000u,0x3d000000u),
518 coefficient(interval,0xbf800000u,0xbfa00000u,0xbfd00000u,0xc0480000u,0xbfd00000u,0xc0480000u,0xc0240000u))...};
519 auto [...h]=std::array{coefficient(interval,0x00000000u,0x00000000u,0x00000000u,0x00000000u,0xb9405fbfu,0x00000000u,0x00000000u)...};
520 ((h=fma(h,t,coefficient(interval,0x00000000u,0x38752140u,0x00000000u,0x00000000u,0x39ab4d5fu,0x00000000u,0x00000000u))),...);
521 ((h=fma(h,t,coefficient(interval,0x00000000u,0xb9183513u,0xb947bd66u,0xb58f6d10u,0xb9c08feeu,0xb59171b1u,0x00000000u))),...);
522 ((h=fma(h,t,coefficient(interval,0x34facb37u,0x398d9ee0u,0x39dfe5a4u,0x36863471u,0x3a2bf5fau,0x3671d749u,0x00000000u))),...);
523 ((h=fma(h,t,coefficient(interval,0xb63a0d2du,0xba2fdf3au,0xba4a3864u,0xb758ec9du,0xbaa65b2au,0xb72757e1u,0xb811c415u))),...);
524 ((h=fma(h,t,coefficient(interval,0x37813497u,0x3ae1130du,0x3ae2b84bu,0x384b3fb7u,0x3b1584c1u,0x380d4deeu,0x38c8e73cu))),...);
525 ((h=fma(h,t,coefficient(interval,0xb8bfb3f8u,0xbb8bc302u,0xbb813c08u,0xb9404721u,0xbb86bc79u,0xb8f4646bu,0xb980a2b8u))),...);
526 ((h=fma(h,t,coefficient(interval,0x3a0e6d24u,0x3c2d6773u,0x3c1129ceu,0x3a356f7bu,0x3bf71905u,0x39d46b54u,0x3a37296bu))),...);
527 ((h=fma(h,t,coefficient(interval,0xbb535f6cu,0xbcd7a178u,0xbca3c0b8u,0xbb2d3d49u,0xbc67da1fu,0xbabdbc02u,0xbb06690eu))),...);
528 ((h=fma(h,t,coefficient(interval,0x3c9d20e4u,0x3d86d45du,0x3d3bcdf4u,0x3c2a7e14u,0x3ce35ff2u,0x3bb1ed8au,0x3bcea86au))),...);
529 ((h=fma(h,t,coefficient(interval,0xbded544du,0xbe2e2df9u,0xbde4f8bcu,0xbd36d397u,0xbd76f5e5u,0xbcb95c19u,0xbcb08499u))),...);
530 ((h=fma(h,t,coefficient(interval,0x3f5c6e3eu,0x3f14c222u,0x3ec662fcu,0x3e9091d7u,0x3e48ced7u,0x3e10d0b5u,0x3de229ecu))),...);
531 auto [...result]=std::array{B::encode(x*h)...};
532 ((result=select(result>U(0x3f800000u),U(0x3f800000u),result) |
533 (word & U(0x80000000u))),...);
534 // Preserve tiny normal inputs and signed zeros without arithmetic, saturate
535 // infinities, and use the scalar contract's canonical NaN result.
536 ((result=select(magnitude>=U(0x41200000u),
537 (word & U(0x80000000u)) | U(0x3f800000u),result)),...);
538 ((result=select(magnitude<=U(0x39800000u),word,result)),...);
539 ((result=select(magnitude>U(0x7f800000u),U(0x7fc00000u),result)),...);
540 return {{B::decode(result)...}};
541 }
542 }
543
544 // Word admission and range reduction feed one copy of the scalar log1p
545 // polynomial. Direct log1p lanes select their original argument; other lanes
546 // select the reduced mantissa. No lane is evaluated by a scalar fallback.
547 // Typed inputs are already canonical. Manual tiny lanes bypass squaring;
548 // hardware tiny lanes may underflow, but return their original words. All
549 // observed intermediates are bounded by tests/log/verify_bounds.py.
550 template <bool OnePlus, bool Hardware, float_register V, std::size_t N>
551 native_flatten constexpr native_inline std::array<V,N> log_ftz(std::array<V,N> const & input) noexcept {
552 using B=detail::fp32_bit_bridge<V>;
553 using U=typename B::bits_type;
554 using I=typename V::template rebind<std::int32_t>;
555 if constexpr (N==0) return {};
556 else {
557 auto const & [...original]=input;
558 auto const [...word]=std::array{B::encode(original)...};
559 auto const [...magnitude]=std::array{(word & U(0x7fffffffu))...};
560 auto const [...valid]=[&] {
561 if constexpr (OnePlus) return std::array{((magnitude < U(0x7f800000u)) &
562 (((word & U(0x80000000u)) == U(0)) | (magnitude < U(0x3f800000u))))...};
563 else return std::array{((word > U(0)) & (word < U(0x7f800000u)))...};
564 }();
565 auto const [...direct]=[&] {
566 if constexpr (OnePlus) return std::array{(valid & (magnitude <=
567 select((word & U(0x80000000u)) != U(0),U(0x3f000000u),U(0x3f800000u))))...};
568 else return std::array<typename V::mask,0>{};
569 }();
570 auto const [...positive]=[&] {
571 if constexpr (OnePlus) return std::array{B::encode(V(1.0f) +
572 B::decode(select(valid & !direct,word,U(0))))...};
573 else return std::array{select(valid,word,U(0x3f800000u))...};
574 }();
575 auto [...exponent]=std::array{(positive.template right<23>() - U(127))...};
576 auto [...mantissa]=std::array{((positive & U(0x007fffffu)) | U(0x3f800000u))...};
577 auto const [...upper]=std::array{(mantissa >= U(0x3fc00000u))...};
578 ((mantissa=mantissa-select(upper,U(0x00800000u),U(0))),...);
579 ((exponent=exponent+select(upper,U(1),U(0))),...);
580 auto const [...reduced]=std::array{(B::decode(mantissa)-V(1.0f))...};
581 auto const [...argument]=[&] {
582 if constexpr (OnePlus && Hardware) return std::array{select(direct,B::decode(word),reduced)...};
583 else if constexpr (OnePlus) return std::array{select(direct,
584 B::decode(select(magnitude>U(0x33000000u),word,U(0))),reduced)...};
585 else return std::array{reduced...};
586 }();
587 auto const [...negative]=std::array{((B::encode(argument) & U(0x80000000u)) != U(0))...};
588 auto coefficient=[](auto mask,unsigned negative_word,unsigned positive_word) noexcept {
589 return B::decode(select(mask,U(negative_word),U(positive_word)));
590 };
591 auto const [...square]=std::array{(argument*argument)...};
592 auto const [...t]=std::array{fma(argument,
593 coefficient(negative,0x40800000u,0x40000000u),
594 coefficient(negative,0x3f800000u,0xbf800000u))...};
595 auto [...h]=std::array{coefficient(negative,0x00000000u,0xb29c7ee2u)...};
596 ((h=fma(h,t,coefficient(negative,0x00000000u,0x3378ea39u))),...);
597 ((h=fma(h,t,coefficient(negative,0xb44f5480u,0xb3faaccbu))),...);
598 ((h=fma(h,t,coefficient(negative,0x352754efu,0x34c9e1cdu))),...);
599 ((h=fma(h,t,coefficient(negative,0xb5bb75dbu,0xb5b13b5eu))),...);
600 ((h=fma(h,t,coefficient(negative,0x369a1c19u,0x36902a0au))),...);
601 ((h=fma(h,t,coefficient(negative,0xb7866f43u,0xb76af011u))),...);
602 ((h=fma(h,t,coefficient(negative,0x3861235au,0x38423d8au))),...);
603 ((h=fma(h,t,coefficient(negative,0xb93e98dfu,0xb9225d51u))),...);
604 ((h=fma(h,t,coefficient(negative,0x3a24a041u,0x3a0988b0u))),...);
605 ((h=fma(h,t,coefficient(negative,0xbb117f6au,0xbaed1a41u))),...);
606 ((h=fma(h,t,coefficient(negative,0x3c04b7c5u,0x3bd13ce0u))),...);
607 ((h=fma(h,t,coefficient(negative,0xbcfda364u,0xbcbeef90u))),...);
608 ((h=fma(h,t,coefficient(negative,0x3e029133u,0x3db786beu))),...);
609 ((h=fma(h,t,coefficient(negative,0xbf1a5884u,0xbec19b82u))),...);
610 auto const [...polynomial]=std::array{fma(square,h,argument)...};
611 auto const [...e]=std::array{convert<float>(I::from_native(
612 __builtin_bit_cast(typename I::native_type, exponent.to_native())))...};
613 auto const [...low]=std::array{fma(e,B::decode(U(0x35bfbe8eu)),polynomial)...};
614 auto [...result]=std::array{B::encode(fma(e,B::decode(U(0x3f317200u)),low))...};
615 if constexpr (OnePlus) {
616 ((result=select(direct,B::encode(polynomial),result)),...);
617 ((result=select(magnitude<=U(0x33000000u),word,result)),...);
618 ((result=select(word==U(0xbf800000u),U(0xff800000u),result)),...);
619 ((result=select(word==U(0x7f800000u),word,result)),...);
620 ((result=select((magnitude>U(0x7f800000u)) |
621 (((word & U(0x80000000u)) != U(0)) & (magnitude>U(0x3f800000u))),U(0x7fc00000u),result)),...);
622 } else {
623 ((result=select(word==U(0x7f800000u),word,result)),...);
624 ((result=select((word & U(0x80000000u)) != U(0),U(0x7fc00000u),result)),...);
625 ((result=select(magnitude==U(0),U(0xff800000u),result)),...);
626 ((result=select(magnitude>U(0x7f800000u),U(0x7fc00000u),result)),...);
627 }
628 return {{B::decode(result)...}};
629 }
630 }
631
632 // The normalized reciprocal and polynomial follow the scalar atan2 graph.
633 // Inputs are canonical FTZ values. Bit-built mantissas stay in [1,2) even
634 // for axis/nonfinite lanes, whose outputs are reconstructed at the end.
635 // Altered coefficient use from SLEEF 3.9.0 atan2kf, under Boost 1.0 below.
636 template<bool Hardware,float_register V,std::size_t N>
637 native_flatten constexpr native_inline std::array<V,N> atan2_ftz(
638 std::array<V,N> const & y_input,std::array<V,N> const & x_input) noexcept {
639 using B=detail::fp32_bit_bridge<V>;
640 using U=typename B::bits_type;
641 using I=typename V::template rebind<std::int32_t>;
642 if constexpr(N==0) return {};
643 else {
644 auto constant=[](unsigned word) noexcept {return B::decode(U(word));};
645 auto integer=[](U word) noexcept {
646 return I::from_native(__builtin_bit_cast(typename I::native_type, word.to_native()));
647 };
648 auto const & [...y]=y_input;
649 auto const & [...x]=x_input;
650 auto const [...yw]=std::array{B::encode(y)...};
651 auto const [...xw]=std::array{B::encode(x)...};
652 auto const [...ay]=std::array{(yw & U(0x7fffffffu))...};
653 auto const [...ax]=std::array{(xw & U(0x7fffffffu))...};
654 auto const [...swap]=std::array{(ay>ax)...};
655 auto const [...negative_x]=std::array{((xw & U(0x80000000u)) != U(0))...};
656 auto const [...a]=std::array{select(swap,ax,ay)...};
657 auto const [...b]=std::array{select(swap,ay,ax)...};
658 auto const [...ma]=std::array{B::decode((a & U(0x007fffffu)) | U(0x3f800000u))...};
659 auto const [...mb]=std::array{((b & U(0x007fffffu)) | U(0x3f800000u))...};
660 auto const [...m]=std::array{B::decode(mb)...};
661 auto [...r]=std::array{B::decode(U(0x7ef311c3u)-mb)...};
662 auto [...e]=std::array{fma(-m,r,V(1.0f))...};
663 ((r=fma(r,e,r)),...);
664 ((e=fma(-m,r,V(1.0f))),...);
665 ((r=fma(r,e,r)),...);
666 ((e=fma(-m,r,V(1.0f))),...);
667 ((r=fma(r,e,r)),...);
668 ((r=ma*r),...);
669 auto const [...word]=std::array{B::encode(r)...};
670 auto const [...fraction]=std::array{(word & U(0x007fffffu))...};
671 auto const [...exponent]=std::array{(word.template right<23>()+
672 a.template right<23>()-b.template right<23>())...};
673 auto [...ratio_word]=std::array{select(integer(exponent)>I(0),
674 exponent.template left<23>() | fraction,
675 select((exponent==U(0)) & (fraction==U(0x007fffffu)),U(0x00800000u),U(0)))...};
676 ((ratio_word=select(ratio_word>U(0x3f800000u),U(0x3f800000u),ratio_word)),...);
677 auto const [...tiny]=std::array{(ratio_word<=U(0x39800000u))...};
678 auto const [...ratio]=std::array{B::decode(ratio_word)...};
679 auto const [...t]=[&] {
680 if constexpr(Hardware) return std::array{ratio...};
681 else return std::array{B::decode(select(tiny,U(0),ratio_word))...};
682 }();
683 auto const [...z]=std::array{(t*t)...};
684 auto [...h]=std::array{fma(constant(0x3b390ccdu),z,constant(0xbc82b80du))...};
685 ((h=fma(h,z,constant(0x3d2e19b6u))),...);
686 ((h=fma(h,z,constant(0xbd995ffau))),...);
687 ((h=fma(h,z,constant(0x3dd9ccf2u))),...);
688 ((h=fma(h,z,constant(0xbe116f9fu))),...);
689 ((h=fma(h,z,constant(0x3e4cb9a7u))),...);
690 ((h=fma(h,z,constant(0xbeaaaa5du))),...);
691 auto const [...q]=std::array{(z*h)...};
692 auto [...angle]=std::array{fma(q,t,ratio)...};
693 ((angle=select(tiny,ratio,angle)),...);
694 ((angle=select(swap,fma(V(-1.0f),angle,constant(0x3fc90fdbu)),angle)),...);
695 ((angle=select(negative_x,fma(V(-1.0f),angle,constant(0x40490fdbu)),angle)),...);
696 auto [...result]=std::array{B::encode(angle)...};
697 auto const [...axis]=std::array{select(negative_x,U(0x40490fdbu),U(0))...};
698 ((result=select(ax==U(0),U(0x3fc90fdbu),result)),...);
699 ((result=select(ax==U(0x7f800000u),axis,result)),...);
700 ((result=select(ay==U(0x7f800000u),select(ax==U(0x7f800000u),
701 select(negative_x,U(0x4016cbe4u),U(0x3f490fdbu)),U(0x3fc90fdbu)),result)),...);
702 ((result=select(ay==U(0),axis,result)),...);
703 ((result=result | (yw & U(0x80000000u))),...);
704 ((result=select((ay>U(0x7f800000u)) | (ax>U(0x7f800000u)),U(0x7fc00000u),result)),...);
705 return {{B::decode(result)...}};
706 }
707 }
708}
709
710/*
711Copyright Naoki Shibata and contributors 2010 - 2025.
712Boost Software License - Version 1.0 - August 17th, 2003
713
714Permission is hereby granted, free of charge, to any person or organization
715obtaining a copy of the software and accompanying documentation covered by
716this license (the "Software") to use, reproduce, display, distribute,
717execute, and transmit the Software, and to prepare derivative works of the
718Software, and to permit third-parties to whom the Software is furnished to
719do so, all subject to the following:
720
721The copyright notices in the Software and this entire statement, including
722the above license grant, this restriction and the following disclaimer,
723must be included in all copies of the Software, in whole or in part, and
724all derivative works of the Software, unless such copies or derivative
725works are solely in the form of machine-executable object code generated by
726a source language processor.
727
728THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
729IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
730FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
731SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
732FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
733ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
734DEALINGS IN THE SOFTWARE.
735*/
ftz::fma
constexpr auto fma(A a, B b, C c) noexcept(std::is_nothrow_constructible_v< detail::ftz32_simd< detail::raw_family_t< A, B, C >, detail::scalar_family_t< A, B, C > >, A & > &&std::is_nothrow_constructible_v< detail::ftz32_simd< detail::raw_family_t< A, B, C >, detail::scalar_family_t< A, B, C > >, B & > &&std::is_nothrow_constructible_v< detail::ftz32_simd< detail::raw_family_t< A, B, C >, detail::scalar_family_t< A, B, C > >, C & >)
Imports raw SIMD operands into the FTZ scalar operand's policy and evaluates fused a*b+c....
Definition simd.h:840