native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
fcma.h
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2#pragma once
4#include "native/config.h"
5#include "native/attributes.h"
6#include "native/isa.h"
7#include "native/arm/detail/register_order.h"
8#if NATIVE_HOST_NEON
9#include <arm_neon.h>
10#endif
11#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
12namespace native::detail {
13
14
15
16
17
18
19
20
21
22 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
23 && (Rotation == 90 || Rotation == 270))
24 native_nodiscard native_inline __attribute__((target("complxnum")))
25 float32x2_t fcadd(float32x2_t a, float32x2_t b) noexcept {
26 a = detail::arm_register_order(a);
27 b = detail::arm_register_order(b);
28 float32x2_t result;
29 asm volatile("fcadd %0.2s, %1.2s, %2.2s, #%3"
30 : "=w"(result) : "w"(a), "w"(b), "i"(Rotation) : "memory");
31 return detail::arm_register_order(result);
32 }
33
34 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
35 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270))
36 native_nodiscard native_inline __attribute__((target("complxnum")))
37 float32x2_t fcmla(float32x2_t acc, float32x2_t a, float32x2_t b) noexcept {
38 a = detail::arm_register_order(a);
39 b = detail::arm_register_order(b);
40 acc = detail::arm_register_order(acc);
41 asm volatile("fcmla %0.2s, %1.2s, %2.2s, #%3"
42 : "+w"(acc) : "w"(a), "w"(b), "i"(Rotation) : "memory");
43 return detail::arm_register_order(acc);
44 }
45
46 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
47 requires(Arch.has(arm_feature::complxnum)
48 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
49 && Lane < 1)
50 native_nodiscard native_inline __attribute__((target("complxnum")))
51 float32x2_t fcmla_lane(float32x2_t acc, float32x2_t a, float32x2_t b) noexcept {
52 return fcmla<Arch, Rotation>(acc, a, b);
53 }
54
55 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
56 requires(Arch.has(arm_feature::complxnum)
57 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
58 && Lane < 2)
59 native_nodiscard native_inline __attribute__((target("complxnum")))
60 float32x2_t fcmla_lane(float32x2_t acc, float32x2_t a, float32x4_t b) noexcept {
61 if constexpr(Lane == 0)
62 return fcmla<Arch, Rotation>(acc, a, vget_low_f32(b));
63 else
64 return fcmla<Arch, Rotation>(acc, a, vget_high_f32(b));
65 }
66
67 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
68 && (Rotation == 90 || Rotation == 270))
69 native_nodiscard native_inline __attribute__((target("complxnum")))
70 float32x4_t fcadd(float32x4_t a, float32x4_t b) noexcept {
71 a = detail::arm_register_order(a);
72 b = detail::arm_register_order(b);
73 float32x4_t result;
74 asm volatile("fcadd %0.4s, %1.4s, %2.4s, #%3"
75 : "=w"(result) : "w"(a), "w"(b), "i"(Rotation) : "memory");
76 return detail::arm_register_order(result);
77 }
78
79 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
80 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270))
81 native_nodiscard native_inline __attribute__((target("complxnum")))
82 float32x4_t fcmla(float32x4_t acc, float32x4_t a, float32x4_t b) noexcept {
83 a = detail::arm_register_order(a);
84 b = detail::arm_register_order(b);
85 acc = detail::arm_register_order(acc);
86 asm volatile("fcmla %0.4s, %1.4s, %2.4s, #%3"
87 : "+w"(acc) : "w"(a), "w"(b), "i"(Rotation) : "memory");
88 return detail::arm_register_order(acc);
89 }
90
91 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
92 requires(Arch.has(arm_feature::complxnum)
93 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
94 && Lane < 1)
95 native_nodiscard native_inline __attribute__((target("complxnum")))
96 float32x4_t fcmla_lane(float32x4_t acc, float32x4_t a, float32x2_t b) noexcept {
97 acc = detail::arm_register_order(acc);
98 a = detail::arm_register_order(a);
99 b = detail::arm_register_order(b);
100 asm volatile("fcmla %0.4s, %1.4s, %2.s[%3], #%4"
101 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane), "i"(Rotation) : "memory");
102 return detail::arm_register_order(acc);
103 }
104
105 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
106 requires(Arch.has(arm_feature::complxnum)
107 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
108 && Lane < 2)
109 native_nodiscard native_inline __attribute__((target("complxnum")))
110 float32x4_t fcmla_lane(float32x4_t acc, float32x4_t a, float32x4_t b) noexcept {
111 acc = detail::arm_register_order(acc);
112 a = detail::arm_register_order(a);
113 b = detail::arm_register_order(b);
114 asm volatile("fcmla %0.4s, %1.4s, %2.s[%3], #%4"
115 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane), "i"(Rotation) : "memory");
116 return detail::arm_register_order(acc);
117 }
118
119 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
120 && (Rotation == 90 || Rotation == 270))
121 native_nodiscard native_inline __attribute__((target("complxnum")))
122 float64x2_t fcadd(float64x2_t a, float64x2_t b) noexcept {
123 a = detail::arm_register_order(a);
124 b = detail::arm_register_order(b);
125 float64x2_t result;
126 asm volatile("fcadd %0.2d, %1.2d, %2.2d, #%3"
127 : "=w"(result) : "w"(a), "w"(b), "i"(Rotation) : "memory");
128 return detail::arm_register_order(result);
129 }
130
131 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum)
132 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270))
133 native_nodiscard native_inline __attribute__((target("complxnum")))
134 float64x2_t fcmla(float64x2_t acc, float64x2_t a, float64x2_t b) noexcept {
135 a = detail::arm_register_order(a);
136 b = detail::arm_register_order(b);
137 acc = detail::arm_register_order(acc);
138 asm volatile("fcmla %0.2d, %1.2d, %2.2d, #%3"
139 : "+w"(acc) : "w"(a), "w"(b), "i"(Rotation) : "memory");
140 return detail::arm_register_order(acc);
141 }
142
143 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
144 && (Rotation == 90 || Rotation == 270))
145 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
146 float16x4_t fcadd(float16x4_t a, float16x4_t b) noexcept {
147 a = detail::arm_register_order(a);
148 b = detail::arm_register_order(b);
149 float16x4_t result;
150 asm volatile("fcadd %0.4h, %1.4h, %2.4h, #%3"
151 : "=w"(result) : "w"(a), "w"(b), "i"(Rotation) : "memory");
152 return detail::arm_register_order(result);
153 }
154
155 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
156 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270))
157 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
158 float16x4_t fcmla(float16x4_t acc, float16x4_t a, float16x4_t b) noexcept {
159 a = detail::arm_register_order(a);
160 b = detail::arm_register_order(b);
161 acc = detail::arm_register_order(acc);
162 asm volatile("fcmla %0.4h, %1.4h, %2.4h, #%3"
163 : "+w"(acc) : "w"(a), "w"(b), "i"(Rotation) : "memory");
164 return detail::arm_register_order(acc);
165 }
166
167 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
168 requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
169 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
170 && Lane < 2)
171 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
172 float16x4_t fcmla_lane(float16x4_t acc, float16x4_t a, float16x4_t b) noexcept {
173 acc = detail::arm_register_order(acc);
174 a = detail::arm_register_order(a);
175 b = detail::arm_register_order(b);
176 asm volatile("fcmla %0.4h, %1.4h, %2.h[%3], #%4"
177 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane), "i"(Rotation) : "memory");
178 return detail::arm_register_order(acc);
179 }
180
181 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
182 requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
183 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
184 && Lane < 4)
185 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
186 float16x4_t fcmla_lane(float16x4_t acc, float16x4_t a, float16x8_t b) noexcept {
187 // The 64-bit half form encodes only the low two complex pairs.
188 if constexpr(Lane >= 2)
189 return fcmla_lane<Arch, Rotation, Lane - 2>(acc, a, vget_high_f16(b));
190 acc = detail::arm_register_order(acc);
191 a = detail::arm_register_order(a);
192 b = detail::arm_register_order(b);
193 asm volatile("fcmla %0.4h, %1.4h, %2.h[%3], #%4"
194 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane % 2), "i"(Rotation) : "memory");
195 return detail::arm_register_order(acc);
196 }
197
198 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
199 && (Rotation == 90 || Rotation == 270))
200 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
201 float16x8_t fcadd(float16x8_t a, float16x8_t b) noexcept {
202 a = detail::arm_register_order(a);
203 b = detail::arm_register_order(b);
204 float16x8_t result;
205 asm volatile("fcadd %0.8h, %1.8h, %2.8h, #%3"
206 : "=w"(result) : "w"(a), "w"(b), "i"(Rotation) : "memory");
207 return detail::arm_register_order(result);
208 }
209
210 template<isa<arm> Arch, unsigned Rotation> requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
211 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270))
212 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
213 float16x8_t fcmla(float16x8_t acc, float16x8_t a, float16x8_t b) noexcept {
214 a = detail::arm_register_order(a);
215 b = detail::arm_register_order(b);
216 acc = detail::arm_register_order(acc);
217 asm volatile("fcmla %0.8h, %1.8h, %2.8h, #%3"
218 : "+w"(acc) : "w"(a), "w"(b), "i"(Rotation) : "memory");
219 return detail::arm_register_order(acc);
220 }
221
222 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
223 requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
224 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
225 && Lane < 2)
226 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
227 float16x8_t fcmla_lane(float16x8_t acc, float16x8_t a, float16x4_t b) noexcept {
228 acc = detail::arm_register_order(acc);
229 a = detail::arm_register_order(a);
230 b = detail::arm_register_order(b);
231 asm volatile("fcmla %0.8h, %1.8h, %2.h[%3], #%4"
232 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane), "i"(Rotation) : "memory");
233 return detail::arm_register_order(acc);
234 }
235
236 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
237 requires(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
238 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
239 && Lane < 4)
240 native_nodiscard native_inline __attribute__((target("complxnum,fullfp16")))
241 float16x8_t fcmla_lane(float16x8_t acc, float16x8_t a, float16x8_t b) noexcept {
242 acc = detail::arm_register_order(acc);
243 a = detail::arm_register_order(a);
244 b = detail::arm_register_order(b);
245 asm volatile("fcmla %0.8h, %1.8h, %2.h[%3], #%4"
246 : "+w"(acc) : "w"(a), "w"(b), "i"(Lane), "i"(Rotation) : "memory");
247 return detail::arm_register_order(acc);
248 }
249
250 // Clang permits implicit same-size NEON vector conversions. Exact deleted
251 // overloads keep missing features and invalid immediates from selecting
252 // an overload of another element format through those conversions.
253 template<isa<arm> Arch, unsigned Rotation>
254 requires(!(Arch.has(arm_feature::complxnum)
255 && (Rotation == 90 || Rotation == 270)))
256 float32x2_t fcadd(float32x2_t, float32x2_t) noexcept = delete;
257
258 template<isa<arm> Arch, unsigned Rotation>
259 requires(!(Arch.has(arm_feature::complxnum)
260 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)))
261 float32x2_t fcmla(float32x2_t, float32x2_t, float32x2_t) noexcept = delete;
262
263 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
264 requires(!(Arch.has(arm_feature::complxnum)
265 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
266 && Lane < 1))
267 float32x2_t fcmla_lane(float32x2_t, float32x2_t, float32x2_t) noexcept = delete;
268
269 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
270 requires(!(Arch.has(arm_feature::complxnum)
271 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
272 && Lane < 2))
273 float32x2_t fcmla_lane(float32x2_t, float32x2_t, float32x4_t) noexcept = delete;
274
275 template<isa<arm> Arch, unsigned Rotation>
276 requires(!(Arch.has(arm_feature::complxnum)
277 && (Rotation == 90 || Rotation == 270)))
278 float32x4_t fcadd(float32x4_t, float32x4_t) noexcept = delete;
279
280 template<isa<arm> Arch, unsigned Rotation>
281 requires(!(Arch.has(arm_feature::complxnum)
282 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)))
283 float32x4_t fcmla(float32x4_t, float32x4_t, float32x4_t) noexcept = delete;
284
285 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
286 requires(!(Arch.has(arm_feature::complxnum)
287 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
288 && Lane < 1))
289 float32x4_t fcmla_lane(float32x4_t, float32x4_t, float32x2_t) noexcept = delete;
290
291 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
292 requires(!(Arch.has(arm_feature::complxnum)
293 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
294 && Lane < 2))
295 float32x4_t fcmla_lane(float32x4_t, float32x4_t, float32x4_t) noexcept = delete;
296
297 template<isa<arm> Arch, unsigned Rotation>
298 requires(!(Arch.has(arm_feature::complxnum)
299 && (Rotation == 90 || Rotation == 270)))
300 float64x2_t fcadd(float64x2_t, float64x2_t) noexcept = delete;
301
302 template<isa<arm> Arch, unsigned Rotation>
303 requires(!(Arch.has(arm_feature::complxnum)
304 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)))
305 float64x2_t fcmla(float64x2_t, float64x2_t, float64x2_t) noexcept = delete;
306
307 template<isa<arm> Arch, unsigned Rotation>
308 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
309 && (Rotation == 90 || Rotation == 270)))
310 float16x4_t fcadd(float16x4_t, float16x4_t) noexcept = delete;
311
312 template<isa<arm> Arch, unsigned Rotation>
313 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
314 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)))
315 float16x4_t fcmla(float16x4_t, float16x4_t, float16x4_t) noexcept = delete;
316
317 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
318 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
319 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
320 && Lane < 2))
321 float16x4_t fcmla_lane(float16x4_t, float16x4_t, float16x4_t) noexcept = delete;
322
323 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
324 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
325 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
326 && Lane < 4))
327 float16x4_t fcmla_lane(float16x4_t, float16x4_t, float16x8_t) noexcept = delete;
328
329 template<isa<arm> Arch, unsigned Rotation>
330 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
331 && (Rotation == 90 || Rotation == 270)))
332 float16x8_t fcadd(float16x8_t, float16x8_t) noexcept = delete;
333
334 template<isa<arm> Arch, unsigned Rotation>
335 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
336 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)))
337 float16x8_t fcmla(float16x8_t, float16x8_t, float16x8_t) noexcept = delete;
338
339 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
340 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
341 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
342 && Lane < 2))
343 float16x8_t fcmla_lane(float16x8_t, float16x8_t, float16x4_t) noexcept = delete;
344
345 template<isa<arm> Arch, unsigned Rotation, unsigned Lane>
346 requires(!(Arch.has(arm_feature::complxnum) && Arch.has(arm_feature::neon_fp16)
347 && (Rotation == 0 || Rotation == 90 || Rotation == 180 || Rotation == 270)
348 && Lane < 4))
349 float16x8_t fcmla_lane(float16x8_t, float16x8_t, float16x8_t) noexcept = delete;
350
351}
352#endif
attributes.h
Compiler attributes for host code, with shader-safe shared modifiers.
native::fcmla_lane
constexpr simd< float, 2, Arch > fcmla_lane(simd< float, 2, Arch > acc, simd< float, 2, Arch > a, simd< float, 2, Arch > b) noexcept
FCMLA using complex pair Lane of b (the lane indexes pairs, not scalars).
Definition native.arm.fcma.ccm:130
native::fcadd
constexpr simd< float, 2, Arch > fcadd(simd< float, 2, Arch > a, simd< float, 2, Arch > b) noexcept
FCADD on 1 binary32 complex pair(s), rotation in degrees.
Definition native.arm.fcma.ccm:79
native::fcmla
constexpr simd< float, 2, Arch > fcmla(simd< float, 2, Arch > acc, simd< float, 2, Arch > a, simd< float, 2, Arch > b) noexcept
FCMLA on 1 binary32 complex pair(s), rotation in degrees.
Definition native.arm.fcma.ccm:100
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native::target
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Definition isa.h:396
native::arm_feature
arm_feature
Independently observable ARM instruction features, using local bit indices.
Definition isa.h:24
native::isa
isa(E) -> isa< detail::family_of< E > >
Deduce the family from a feature enum; explicit isa<> always names the host family.