native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
native.x86.ifma.ccm
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2module;
3
4#include "native/isa_import.h"
5#include "native/x86/ifma.h"
6
7export module native.x86.ifma;
8export import native.x86.features;
9export import native.simd;
10
11// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
12
13#if NATIVE_HOST_X86 || defined(NATIVE_DOXYGEN)
14export namespace native {
23
25 template<isa<x86> Arch> requires(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx))
26 native_nodiscard native_inline native_const native_target("avxifma")
27 constexpr simd<std::uint64_t, 2, Arch> madd52lo(
28 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
29 if consteval {
30 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
31 } else {
32 return simd<std::uint64_t, 2, Arch>::from_native(
33 detail::x86_ifma::madd52lo<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
34 }
35 }
36
38 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) &&
39 Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
40 Arch.has(x86_feature::avx512vl))
41 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
42 constexpr simd<std::uint64_t, 2, Arch> madd52lo(
43 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
44 if consteval {
45 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
46 } else {
47 return simd<std::uint64_t, 2, Arch>::from_native(
48 detail::x86_ifma::madd52lo<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
49 }
50 }
51
53 template<isa<x86> Arch> requires(!((Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) || (Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
54 Arch.has(x86_feature::avx512vl))) &&
55 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
56 native_nodiscard consteval simd<std::uint64_t, 2, Arch> madd52lo(
57 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
58 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
59 }
60
62 template<isa<x86> Arch> requires(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx))
63 native_nodiscard native_inline native_const native_target("avxifma")
64 constexpr simd<std::uint64_t, 4, Arch> madd52lo(
65 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
66 if consteval {
67 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
68 } else {
69 return simd<std::uint64_t, 4, Arch>::from_native(
70 detail::x86_ifma::madd52lo<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
71 }
72 }
73
75 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) &&
76 Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
77 Arch.has(x86_feature::avx512vl))
78 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
79 constexpr simd<std::uint64_t, 4, Arch> madd52lo(
80 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
81 if consteval {
82 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
83 } else {
84 return simd<std::uint64_t, 4, Arch>::from_native(
85 detail::x86_ifma::madd52lo<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
86 }
87 }
88
90 template<isa<x86> Arch> requires(!((Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) || (Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
91 Arch.has(x86_feature::avx512vl))) &&
92 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
93 native_nodiscard consteval simd<std::uint64_t, 4, Arch> madd52lo(
94 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
95 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
96 }
97
99 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
100 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
101 constexpr simd<std::uint64_t, 8, Arch> madd52lo(
102 simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
103 if consteval {
104 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
105 } else {
106 return simd<std::uint64_t, 8, Arch>::from_native(
107 detail::x86_ifma::madd52lo<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
108 }
109 }
110
112 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
113 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
114 native_nodiscard consteval simd<std::uint64_t, 8, Arch> madd52lo(
115 simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
116 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, ~std::uint64_t{0}, false);
117 }
118
120 template<isa<x86> Arch, class... Args>
121 void madd52lo(Args...) = delete;
122
124 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
125 Arch.has(x86_feature::avx512vl))
126 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
127 constexpr simd<std::uint64_t, 2, Arch> mask_madd52lo(
128 simd<std::uint64_t, 2, Arch> accumulator, predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
129 if consteval {
130 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
131 } else {
132 return simd<std::uint64_t, 2, Arch>::from_native(
133 detail::x86_ifma::mask_madd52lo<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
134 }
135 }
136
138 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
139 Arch.has(x86_feature::avx512vl)) &&
140 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
141 native_nodiscard consteval simd<std::uint64_t, 2, Arch> mask_madd52lo(
142 simd<std::uint64_t, 2, Arch> accumulator, predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
143 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
144 }
145
147 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
148 Arch.has(x86_feature::avx512vl))
149 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
150 constexpr simd<std::uint64_t, 4, Arch> mask_madd52lo(
151 simd<std::uint64_t, 4, Arch> accumulator, predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
152 if consteval {
153 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
154 } else {
155 return simd<std::uint64_t, 4, Arch>::from_native(
156 detail::x86_ifma::mask_madd52lo<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
157 }
158 }
159
161 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
162 Arch.has(x86_feature::avx512vl)) &&
163 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
164 native_nodiscard consteval simd<std::uint64_t, 4, Arch> mask_madd52lo(
165 simd<std::uint64_t, 4, Arch> accumulator, predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
166 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
167 }
168
170 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
171 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
172 constexpr simd<std::uint64_t, 8, Arch> mask_madd52lo(
173 simd<std::uint64_t, 8, Arch> accumulator, predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
174 if consteval {
175 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
176 } else {
177 return simd<std::uint64_t, 8, Arch>::from_native(
178 detail::x86_ifma::mask_madd52lo<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
179 }
180 }
181
183 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
184 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
185 native_nodiscard consteval simd<std::uint64_t, 8, Arch> mask_madd52lo(
186 simd<std::uint64_t, 8, Arch> accumulator, predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
187 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), false);
188 }
189
191 template<isa<x86> Arch, class... Args>
192 void mask_madd52lo(Args...) = delete;
193
195 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
196 Arch.has(x86_feature::avx512vl))
197 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
198 constexpr simd<std::uint64_t, 2, Arch> maskz_madd52lo(
199 predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
200 if consteval {
201 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
202 } else {
203 return simd<std::uint64_t, 2, Arch>::from_native(
204 detail::x86_ifma::maskz_madd52lo<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
205 }
206 }
207
209 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
210 Arch.has(x86_feature::avx512vl)) &&
211 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
212 native_nodiscard consteval simd<std::uint64_t, 2, Arch> maskz_madd52lo(
213 predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
214 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
215 }
216
218 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
219 Arch.has(x86_feature::avx512vl))
220 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
221 constexpr simd<std::uint64_t, 4, Arch> maskz_madd52lo(
222 predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
223 if consteval {
224 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
225 } else {
226 return simd<std::uint64_t, 4, Arch>::from_native(
227 detail::x86_ifma::maskz_madd52lo<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
228 }
229 }
230
232 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
233 Arch.has(x86_feature::avx512vl)) &&
234 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
235 native_nodiscard consteval simd<std::uint64_t, 4, Arch> maskz_madd52lo(
236 predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
237 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
238 }
239
241 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
242 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
243 constexpr simd<std::uint64_t, 8, Arch> maskz_madd52lo(
244 predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
245 if consteval {
246 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
247 } else {
248 return simd<std::uint64_t, 8, Arch>::from_native(
249 detail::x86_ifma::maskz_madd52lo<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
250 }
251 }
252
254 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
255 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
256 native_nodiscard consteval simd<std::uint64_t, 8, Arch> maskz_madd52lo(
257 predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
258 return detail::x86_ifma_constant::accumulate<false>(accumulator, a, b, mask.to_bitset(), true);
259 }
260
262 template<isa<x86> Arch, class... Args>
263 void maskz_madd52lo(Args...) = delete;
264
266 template<isa<x86> Arch> requires(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx))
267 native_nodiscard native_inline native_const native_target("avxifma")
268 constexpr simd<std::uint64_t, 2, Arch> madd52hi(
269 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
270 if consteval {
271 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
272 } else {
273 return simd<std::uint64_t, 2, Arch>::from_native(
274 detail::x86_ifma::madd52hi<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
275 }
276 }
277
279 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) &&
280 Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
281 Arch.has(x86_feature::avx512vl))
282 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
283 constexpr simd<std::uint64_t, 2, Arch> madd52hi(
284 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
285 if consteval {
286 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
287 } else {
288 return simd<std::uint64_t, 2, Arch>::from_native(
289 detail::x86_ifma::madd52hi<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
290 }
291 }
292
294 template<isa<x86> Arch> requires(!((Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) || (Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
295 Arch.has(x86_feature::avx512vl))) &&
296 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
297 native_nodiscard consteval simd<std::uint64_t, 2, Arch> madd52hi(
298 simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
299 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
300 }
301
303 template<isa<x86> Arch> requires(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx))
304 native_nodiscard native_inline native_const native_target("avxifma")
305 constexpr simd<std::uint64_t, 4, Arch> madd52hi(
306 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
307 if consteval {
308 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
309 } else {
310 return simd<std::uint64_t, 4, Arch>::from_native(
311 detail::x86_ifma::madd52hi<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
312 }
313 }
314
316 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) &&
317 Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
318 Arch.has(x86_feature::avx512vl))
319 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
320 constexpr simd<std::uint64_t, 4, Arch> madd52hi(
321 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
322 if consteval {
323 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
324 } else {
325 return simd<std::uint64_t, 4, Arch>::from_native(
326 detail::x86_ifma::madd52hi<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
327 }
328 }
329
331 template<isa<x86> Arch> requires(!((Arch.has(x86_feature::avxifma) && Arch.has(x86_feature::avx)) || (Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
332 Arch.has(x86_feature::avx512vl))) &&
333 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
334 native_nodiscard consteval simd<std::uint64_t, 4, Arch> madd52hi(
335 simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
336 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
337 }
338
340 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
341 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
342 constexpr simd<std::uint64_t, 8, Arch> madd52hi(
343 simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
344 if consteval {
345 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
346 } else {
347 return simd<std::uint64_t, 8, Arch>::from_native(
348 detail::x86_ifma::madd52hi<Arch>(accumulator.to_native(), a.to_native(), b.to_native()));
349 }
350 }
351
353 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
354 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
355 native_nodiscard consteval simd<std::uint64_t, 8, Arch> madd52hi(
356 simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
357 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, ~std::uint64_t{0}, false);
358 }
359
361 template<isa<x86> Arch, class... Args>
362 void madd52hi(Args...) = delete;
363
365 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
366 Arch.has(x86_feature::avx512vl))
367 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
368 constexpr simd<std::uint64_t, 2, Arch> mask_madd52hi(
369 simd<std::uint64_t, 2, Arch> accumulator, predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
370 if consteval {
371 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
372 } else {
373 return simd<std::uint64_t, 2, Arch>::from_native(
374 detail::x86_ifma::mask_madd52hi<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
375 }
376 }
377
379 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
380 Arch.has(x86_feature::avx512vl)) &&
381 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
382 native_nodiscard consteval simd<std::uint64_t, 2, Arch> mask_madd52hi(
383 simd<std::uint64_t, 2, Arch> accumulator, predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
384 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
385 }
386
388 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
389 Arch.has(x86_feature::avx512vl))
390 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
391 constexpr simd<std::uint64_t, 4, Arch> mask_madd52hi(
392 simd<std::uint64_t, 4, Arch> accumulator, predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
393 if consteval {
394 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
395 } else {
396 return simd<std::uint64_t, 4, Arch>::from_native(
397 detail::x86_ifma::mask_madd52hi<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
398 }
399 }
400
402 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
403 Arch.has(x86_feature::avx512vl)) &&
404 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
405 native_nodiscard consteval simd<std::uint64_t, 4, Arch> mask_madd52hi(
406 simd<std::uint64_t, 4, Arch> accumulator, predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
407 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
408 }
409
411 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
412 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
413 constexpr simd<std::uint64_t, 8, Arch> mask_madd52hi(
414 simd<std::uint64_t, 8, Arch> accumulator, predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
415 if consteval {
416 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
417 } else {
418 return simd<std::uint64_t, 8, Arch>::from_native(
419 detail::x86_ifma::mask_madd52hi<Arch>(accumulator.to_native(), static_cast<__mmask8>(mask.to_bitset()), a.to_native(), b.to_native()));
420 }
421 }
422
424 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
425 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
426 native_nodiscard consteval simd<std::uint64_t, 8, Arch> mask_madd52hi(
427 simd<std::uint64_t, 8, Arch> accumulator, predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
428 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), false);
429 }
430
432 template<isa<x86> Arch, class... Args>
433 void mask_madd52hi(Args...) = delete;
434
436 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
437 Arch.has(x86_feature::avx512vl))
438 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
439 constexpr simd<std::uint64_t, 2, Arch> maskz_madd52hi(
440 predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
441 if consteval {
442 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
443 } else {
444 return simd<std::uint64_t, 2, Arch>::from_native(
445 detail::x86_ifma::maskz_madd52hi<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
446 }
447 }
448
450 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
451 Arch.has(x86_feature::avx512vl)) &&
452 requires { sizeof(simd<std::uint64_t, 2, Arch>); })
453 native_nodiscard consteval simd<std::uint64_t, 2, Arch> maskz_madd52hi(
454 predicate<2, Arch> mask, simd<std::uint64_t, 2, Arch> accumulator, simd<std::uint64_t, 2, Arch> a, simd<std::uint64_t, 2, Arch> b) noexcept {
455 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
456 }
457
459 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
460 Arch.has(x86_feature::avx512vl))
461 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma,avx512vl")
462 constexpr simd<std::uint64_t, 4, Arch> maskz_madd52hi(
463 predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
464 if consteval {
465 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
466 } else {
467 return simd<std::uint64_t, 4, Arch>::from_native(
468 detail::x86_ifma::maskz_madd52hi<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
469 }
470 }
471
473 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma) &&
474 Arch.has(x86_feature::avx512vl)) &&
475 requires { sizeof(simd<std::uint64_t, 4, Arch>); })
476 native_nodiscard consteval simd<std::uint64_t, 4, Arch> maskz_madd52hi(
477 predicate<4, Arch> mask, simd<std::uint64_t, 4, Arch> accumulator, simd<std::uint64_t, 4, Arch> a, simd<std::uint64_t, 4, Arch> b) noexcept {
478 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
479 }
480
482 template<isa<x86> Arch> requires(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma))
483 native_nodiscard native_inline native_const native_target("avx512f,avx512ifma")
484 constexpr simd<std::uint64_t, 8, Arch> maskz_madd52hi(
485 predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
486 if consteval {
487 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
488 } else {
489 return simd<std::uint64_t, 8, Arch>::from_native(
490 detail::x86_ifma::maskz_madd52hi<Arch>(static_cast<__mmask8>(mask.to_bitset()), accumulator.to_native(), a.to_native(), b.to_native()));
491 }
492 }
493
495 template<isa<x86> Arch> requires(!(Arch.has(x86_feature::avx512f) && Arch.has(x86_feature::avx512ifma)) &&
496 requires { sizeof(simd<std::uint64_t, 8, Arch>); })
497 native_nodiscard consteval simd<std::uint64_t, 8, Arch> maskz_madd52hi(
498 predicate<8, Arch> mask, simd<std::uint64_t, 8, Arch> accumulator, simd<std::uint64_t, 8, Arch> a, simd<std::uint64_t, 8, Arch> b) noexcept {
499 return detail::x86_ifma_constant::accumulate<true>(accumulator, a, b, mask.to_bitset(), true);
500 }
501
503 template<isa<x86> Arch, class... Args>
504 void maskz_madd52hi(Args...) = delete;
505
507}
508#endif
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native_const
#define native_const
[[const]] is not const
Definition attributes.h:108
native_target
#define native_target(x)
this indicates a required feature set for the current multiversioned function.
Definition attributes.h:476
native::mask
typename mask_traits< std::remove_cvref_t< T > >::type mask
Definition mask_traits.h:22
native::mask_madd52lo
constexpr simd< std::uint64_t, 2, Arch > mask_madd52lo(simd< std::uint64_t, 2, Arch > accumulator, predicate< 2, Arch > mask, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate low 52-bit products in 2 lanes; inactive lanes retain the accumulator.
Definition native.x86.ifma.ccm:127
native::maskz_madd52lo
constexpr simd< std::uint64_t, 2, Arch > maskz_madd52lo(predicate< 2, Arch > mask, simd< std::uint64_t, 2, Arch > accumulator, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate low 52-bit products in 2 lanes; inactive lanes become zero.
Definition native.x86.ifma.ccm:198
native::maskz_madd52hi
constexpr simd< std::uint64_t, 2, Arch > maskz_madd52hi(predicate< 2, Arch > mask, simd< std::uint64_t, 2, Arch > accumulator, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate high 52-bit products in 2 lanes; inactive lanes become zero.
Definition native.x86.ifma.ccm:439
native::madd52hi
constexpr simd< std::uint64_t, 2, Arch > madd52hi(simd< std::uint64_t, 2, Arch > accumulator, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate high 52-bit products in 2 lanes; unmasked.
Definition native.x86.ifma.ccm:268
native::mask_madd52hi
constexpr simd< std::uint64_t, 2, Arch > mask_madd52hi(simd< std::uint64_t, 2, Arch > accumulator, predicate< 2, Arch > mask, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate high 52-bit products in 2 lanes; inactive lanes retain the accumulator.
Definition native.x86.ifma.ccm:368
native::madd52lo
constexpr simd< std::uint64_t, 2, Arch > madd52lo(simd< std::uint64_t, 2, Arch > accumulator, simd< std::uint64_t, 2, Arch > a, simd< std::uint64_t, 2, Arch > b) noexcept
Accumulate low 52-bit products in 2 lanes; unmasked.
Definition native.x86.ifma.ccm:27
native.simd
native.x86.features
native.x86.ifma
native
Architecture-tagged vectors, register packs and supporting value types. Native arithmetic follows its...
Definition native.arm.aes.ccm:10
std
Standard-library adaptations documented here for SIMD value types.
native::simd
Omitted architecture arguments use the native.simd provider's baseline.
Definition native.simd.ccm:19