native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
fp16fml.h
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2#pragma once
4#include "native/config.h"
5#include "native/attributes.h"
6#include "native/isa.h"
7#include "native/arm/detail/register_order.h"
8#if NATIVE_HOST_NEON
9#include <arm_neon.h>
10#endif
11#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
12namespace native::detail {
13
14
15
16
17
18
19
20 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
21 native_nodiscard native_inline __attribute__((target("fp16fml")))
22 float32x2_t fmlal(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
23 acc = detail::arm_register_order(acc);
24 a = detail::arm_register_order(a);
25 b = detail::arm_register_order(b);
26 asm volatile("fmlal %0.2s, %1.2h, %2.2h"
27 : "+w"(acc) : "w"(a), "w"(b) : "memory");
28 return detail::arm_register_order(acc);
29 }
30
31 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
32 native_nodiscard native_inline __attribute__((target("fp16fml")))
33 float32x2_t fmlal_lane(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
34 acc = detail::arm_register_order(acc);
35 a = detail::arm_register_order(a);
36 auto source = detail::arm_register_order(vcombine_f16(b, b));
37 asm volatile("fmlal %0.2s, %1.2h, %2.h[%3]"
38 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
39 return detail::arm_register_order(acc);
40 }
41
42 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
43 native_nodiscard native_inline __attribute__((target("fp16fml")))
44 float32x2_t fmlal_lane(float32x2_t acc, float16x4_t a, float16x8_t b) noexcept {
45 acc = detail::arm_register_order(acc);
46 a = detail::arm_register_order(a);
47 auto source = detail::arm_register_order(b);
48 asm volatile("fmlal %0.2s, %1.2h, %2.h[%3]"
49 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
50 return detail::arm_register_order(acc);
51 }
52
53 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
54 native_nodiscard native_inline __attribute__((target("fp16fml")))
55 float32x4_t fmlal(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
56 acc = detail::arm_register_order(acc);
57 a = detail::arm_register_order(a);
58 b = detail::arm_register_order(b);
59 asm volatile("fmlal %0.4s, %1.4h, %2.4h"
60 : "+w"(acc) : "w"(a), "w"(b) : "memory");
61 return detail::arm_register_order(acc);
62 }
63
64 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
65 native_nodiscard native_inline __attribute__((target("fp16fml")))
66 float32x4_t fmlal_lane(float32x4_t acc, float16x8_t a, float16x4_t b) noexcept {
67 acc = detail::arm_register_order(acc);
68 a = detail::arm_register_order(a);
69 auto source = detail::arm_register_order(vcombine_f16(b, b));
70 asm volatile("fmlal %0.4s, %1.4h, %2.h[%3]"
71 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
72 return detail::arm_register_order(acc);
73 }
74
75 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
76 native_nodiscard native_inline __attribute__((target("fp16fml")))
77 float32x4_t fmlal_lane(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
78 acc = detail::arm_register_order(acc);
79 a = detail::arm_register_order(a);
80 auto source = detail::arm_register_order(b);
81 asm volatile("fmlal %0.4s, %1.4h, %2.h[%3]"
82 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
83 return detail::arm_register_order(acc);
84 }
85
86 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
87 native_nodiscard native_inline __attribute__((target("fp16fml")))
88 float32x2_t fmlal2(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
89 acc = detail::arm_register_order(acc);
90 a = detail::arm_register_order(a);
91 b = detail::arm_register_order(b);
92 asm volatile("fmlal2 %0.2s, %1.2h, %2.2h"
93 : "+w"(acc) : "w"(a), "w"(b) : "memory");
94 return detail::arm_register_order(acc);
95 }
96
97 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
98 native_nodiscard native_inline __attribute__((target("fp16fml")))
99 float32x2_t fmlal2_lane(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
100 acc = detail::arm_register_order(acc);
101 a = detail::arm_register_order(a);
102 auto source = detail::arm_register_order(vcombine_f16(b, b));
103 asm volatile("fmlal2 %0.2s, %1.2h, %2.h[%3]"
104 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
105 return detail::arm_register_order(acc);
106 }
107
108 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
109 native_nodiscard native_inline __attribute__((target("fp16fml")))
110 float32x2_t fmlal2_lane(float32x2_t acc, float16x4_t a, float16x8_t b) noexcept {
111 acc = detail::arm_register_order(acc);
112 a = detail::arm_register_order(a);
113 auto source = detail::arm_register_order(b);
114 asm volatile("fmlal2 %0.2s, %1.2h, %2.h[%3]"
115 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
116 return detail::arm_register_order(acc);
117 }
118
119 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
120 native_nodiscard native_inline __attribute__((target("fp16fml")))
121 float32x4_t fmlal2(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
122 acc = detail::arm_register_order(acc);
123 a = detail::arm_register_order(a);
124 b = detail::arm_register_order(b);
125 asm volatile("fmlal2 %0.4s, %1.4h, %2.4h"
126 : "+w"(acc) : "w"(a), "w"(b) : "memory");
127 return detail::arm_register_order(acc);
128 }
129
130 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
131 native_nodiscard native_inline __attribute__((target("fp16fml")))
132 float32x4_t fmlal2_lane(float32x4_t acc, float16x8_t a, float16x4_t b) noexcept {
133 acc = detail::arm_register_order(acc);
134 a = detail::arm_register_order(a);
135 auto source = detail::arm_register_order(vcombine_f16(b, b));
136 asm volatile("fmlal2 %0.4s, %1.4h, %2.h[%3]"
137 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
138 return detail::arm_register_order(acc);
139 }
140
141 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
142 native_nodiscard native_inline __attribute__((target("fp16fml")))
143 float32x4_t fmlal2_lane(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
144 acc = detail::arm_register_order(acc);
145 a = detail::arm_register_order(a);
146 auto source = detail::arm_register_order(b);
147 asm volatile("fmlal2 %0.4s, %1.4h, %2.h[%3]"
148 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
149 return detail::arm_register_order(acc);
150 }
151
152 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
153 native_nodiscard native_inline __attribute__((target("fp16fml")))
154 float32x2_t fmlsl(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
155 acc = detail::arm_register_order(acc);
156 a = detail::arm_register_order(a);
157 b = detail::arm_register_order(b);
158 asm volatile("fmlsl %0.2s, %1.2h, %2.2h"
159 : "+w"(acc) : "w"(a), "w"(b) : "memory");
160 return detail::arm_register_order(acc);
161 }
162
163 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
164 native_nodiscard native_inline __attribute__((target("fp16fml")))
165 float32x2_t fmlsl_lane(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
166 acc = detail::arm_register_order(acc);
167 a = detail::arm_register_order(a);
168 auto source = detail::arm_register_order(vcombine_f16(b, b));
169 asm volatile("fmlsl %0.2s, %1.2h, %2.h[%3]"
170 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
171 return detail::arm_register_order(acc);
172 }
173
174 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
175 native_nodiscard native_inline __attribute__((target("fp16fml")))
176 float32x2_t fmlsl_lane(float32x2_t acc, float16x4_t a, float16x8_t b) noexcept {
177 acc = detail::arm_register_order(acc);
178 a = detail::arm_register_order(a);
179 auto source = detail::arm_register_order(b);
180 asm volatile("fmlsl %0.2s, %1.2h, %2.h[%3]"
181 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
182 return detail::arm_register_order(acc);
183 }
184
185 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
186 native_nodiscard native_inline __attribute__((target("fp16fml")))
187 float32x4_t fmlsl(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
188 acc = detail::arm_register_order(acc);
189 a = detail::arm_register_order(a);
190 b = detail::arm_register_order(b);
191 asm volatile("fmlsl %0.4s, %1.4h, %2.4h"
192 : "+w"(acc) : "w"(a), "w"(b) : "memory");
193 return detail::arm_register_order(acc);
194 }
195
196 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
197 native_nodiscard native_inline __attribute__((target("fp16fml")))
198 float32x4_t fmlsl_lane(float32x4_t acc, float16x8_t a, float16x4_t b) noexcept {
199 acc = detail::arm_register_order(acc);
200 a = detail::arm_register_order(a);
201 auto source = detail::arm_register_order(vcombine_f16(b, b));
202 asm volatile("fmlsl %0.4s, %1.4h, %2.h[%3]"
203 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
204 return detail::arm_register_order(acc);
205 }
206
207 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
208 native_nodiscard native_inline __attribute__((target("fp16fml")))
209 float32x4_t fmlsl_lane(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
210 acc = detail::arm_register_order(acc);
211 a = detail::arm_register_order(a);
212 auto source = detail::arm_register_order(b);
213 asm volatile("fmlsl %0.4s, %1.4h, %2.h[%3]"
214 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
215 return detail::arm_register_order(acc);
216 }
217
218 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
219 native_nodiscard native_inline __attribute__((target("fp16fml")))
220 float32x2_t fmlsl2(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
221 acc = detail::arm_register_order(acc);
222 a = detail::arm_register_order(a);
223 b = detail::arm_register_order(b);
224 asm volatile("fmlsl2 %0.2s, %1.2h, %2.2h"
225 : "+w"(acc) : "w"(a), "w"(b) : "memory");
226 return detail::arm_register_order(acc);
227 }
228
229 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
230 native_nodiscard native_inline __attribute__((target("fp16fml")))
231 float32x2_t fmlsl2_lane(float32x2_t acc, float16x4_t a, float16x4_t b) noexcept {
232 acc = detail::arm_register_order(acc);
233 a = detail::arm_register_order(a);
234 auto source = detail::arm_register_order(vcombine_f16(b, b));
235 asm volatile("fmlsl2 %0.2s, %1.2h, %2.h[%3]"
236 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
237 return detail::arm_register_order(acc);
238 }
239
240 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
241 native_nodiscard native_inline __attribute__((target("fp16fml")))
242 float32x2_t fmlsl2_lane(float32x2_t acc, float16x4_t a, float16x8_t b) noexcept {
243 acc = detail::arm_register_order(acc);
244 a = detail::arm_register_order(a);
245 auto source = detail::arm_register_order(b);
246 asm volatile("fmlsl2 %0.2s, %1.2h, %2.h[%3]"
247 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
248 return detail::arm_register_order(acc);
249 }
250
251 template<isa<arm> Arch> requires(Arch.has(arm_feature::fp16fml))
252 native_nodiscard native_inline __attribute__((target("fp16fml")))
253 float32x4_t fmlsl2(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
254 acc = detail::arm_register_order(acc);
255 a = detail::arm_register_order(a);
256 b = detail::arm_register_order(b);
257 asm volatile("fmlsl2 %0.4s, %1.4h, %2.4h"
258 : "+w"(acc) : "w"(a), "w"(b) : "memory");
259 return detail::arm_register_order(acc);
260 }
261
262 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 4)
263 native_nodiscard native_inline __attribute__((target("fp16fml")))
264 float32x4_t fmlsl2_lane(float32x4_t acc, float16x8_t a, float16x4_t b) noexcept {
265 acc = detail::arm_register_order(acc);
266 a = detail::arm_register_order(a);
267 auto source = detail::arm_register_order(vcombine_f16(b, b));
268 asm volatile("fmlsl2 %0.4s, %1.4h, %2.h[%3]"
269 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
270 return detail::arm_register_order(acc);
271 }
272
273 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::fp16fml) && Lane < 8)
274 native_nodiscard native_inline __attribute__((target("fp16fml")))
275 float32x4_t fmlsl2_lane(float32x4_t acc, float16x8_t a, float16x8_t b) noexcept {
276 acc = detail::arm_register_order(acc);
277 a = detail::arm_register_order(a);
278 auto source = detail::arm_register_order(b);
279 asm volatile("fmlsl2 %0.4s, %1.4h, %2.h[%3]"
280 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
281 return detail::arm_register_order(acc);
282 }
283
284}
285#endif
attributes.h
Compiler attributes for host code, with shader-safe shared modifiers.
native::fmlal
constexpr simd< float, 2, Arch > fmlal(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
Add products from the low 2 half lanes of a and b.
Definition native.arm.fp16fml.ccm:48
native::fmlsl
constexpr simd< float, 2, Arch > fmlsl(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
Subtract products from the low 2 half lanes of a and b.
Definition native.arm.fp16fml.ccm:360
native::fmlsl2_lane
constexpr simd< float, 2, Arch > fmlsl2_lane(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
FMLSL2 with b[Lane] broadcast; selects the high 2 lanes of a.
Definition native.arm.fp16fml.ccm:542
native::fmlal_lane
constexpr simd< float, 2, Arch > fmlal_lane(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
FMLAL with b[Lane] broadcast; selects the low 2 lanes of a.
Definition native.arm.fp16fml.ccm:74
native::fmlal2
constexpr simd< float, 2, Arch > fmlal2(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
Add products from the high 2 half lanes of a and b.
Definition native.arm.fp16fml.ccm:204
native::fmlsl2
constexpr simd< float, 2, Arch > fmlsl2(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
Subtract products from the high 2 half lanes of a and b.
Definition native.arm.fp16fml.ccm:516
native::fmlal2_lane
constexpr simd< float, 2, Arch > fmlal2_lane(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
FMLAL2 with b[Lane] broadcast; selects the high 2 lanes of a.
Definition native.arm.fp16fml.ccm:230
native::fmlsl_lane
constexpr simd< float, 2, Arch > fmlsl_lane(simd< float, 2, Arch > acc, simd< fp16, 4, Arch > a, simd< fp16, 4, Arch > b) noexcept
FMLSL with b[Lane] broadcast; selects the low 2 lanes of a.
Definition native.arm.fp16fml.ccm:386
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native::target
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Definition isa.h:396