native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
native.arm.i8mm.ccm
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2module;
3#include "native/isa_import.h"
4#include "native/arm/detail/integer_constant.h"
5#include "native/arm/i8mm.h"
6export module native.arm.i8mm;
7export import native.arm.features;
8export import native.simd;
9export {
10#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
11namespace native {
22
23 // All vector operands share Arch; native registers remain implementation details.
25 template<isa<arm> Arch> requires(Arch.has(arm_feature::i8mm))
26 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
27 constexpr simd<std::int32_t, 4, Arch> smmla(
28 simd<std::int32_t, 4, Arch> acc,
29 simd<std::int8_t, 16, Arch> a,
30 simd<std::int8_t, 16, Arch> b) noexcept {
31 if consteval { return detail::arm_constant::dot<true, -1>(acc, a, b); }
32 else {
33 auto result = detail::smmla<Arch>(
34 __builtin_bit_cast(int32x4_t, acc.to_native()),
35 __builtin_bit_cast(int8x16_t, a.to_native()),
36 __builtin_bit_cast(int8x16_t, b.to_native()));
38 }
39 }
40
42 template<isa<arm> Arch> requires(!Arch.has(arm_feature::i8mm) && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
46 simd<std::int8_t, 16, Arch> b) noexcept {
47 return detail::arm_constant::dot<true, -1>(acc, a, b);
48 }
49
51 template<isa<arm> Arch> requires(Arch.has(arm_feature::i8mm))
52 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
53 constexpr simd<std::uint32_t, 4, Arch> ummla(
54 simd<std::uint32_t, 4, Arch> acc,
55 simd<std::uint8_t, 16, Arch> a,
56 simd<std::uint8_t, 16, Arch> b) noexcept {
57 if consteval { return detail::arm_constant::dot<true, -1>(acc, a, b); }
58 else {
59 auto result = detail::ummla<Arch>(
60 __builtin_bit_cast(uint32x4_t, acc.to_native()),
61 __builtin_bit_cast(uint8x16_t, a.to_native()),
62 __builtin_bit_cast(uint8x16_t, b.to_native()));
64 }
65 }
66
68 template<isa<arm> Arch> requires(!Arch.has(arm_feature::i8mm) && requires { sizeof(simd<std::uint32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
73 return detail::arm_constant::dot<true, -1>(acc, a, b);
74 }
75
77 template<isa<arm> Arch> requires(Arch.has(arm_feature::i8mm))
78 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
79 constexpr simd<std::int32_t, 4, Arch> usmmla(
80 simd<std::int32_t, 4, Arch> acc,
81 simd<std::uint8_t, 16, Arch> a,
82 simd<std::int8_t, 16, Arch> b) noexcept {
83 if consteval { return detail::arm_constant::dot<true, -1>(acc, a, b); }
84 else {
85 auto result = detail::usmmla<Arch>(
86 __builtin_bit_cast(int32x4_t, acc.to_native()),
87 __builtin_bit_cast(uint8x16_t, a.to_native()),
88 __builtin_bit_cast(int8x16_t, b.to_native()));
90 }
91 }
92
94 template<isa<arm> Arch> requires(!Arch.has(arm_feature::i8mm) && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
98 simd<std::int8_t, 16, Arch> b) noexcept {
99 return detail::arm_constant::dot<true, -1>(acc, a, b);
100 }
101
103 template<isa<arm> Arch> requires(Arch.has(arm_feature::i8mm))
104 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
105 constexpr simd<std::int32_t, 2, Arch> usdot(
106 simd<std::int32_t, 2, Arch> acc,
107 simd<std::uint8_t, 8, Arch> a,
108 simd<std::int8_t, 8, Arch> b) noexcept {
109 if consteval { return detail::arm_constant::dot<false, -1>(acc, a, b); }
110 else {
111 auto result = detail::usdot<Arch>(
112 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
113 __builtin_bit_cast(uint8x8_t, a.to_native()),
114 __builtin_bit_cast(int8x8_t, b.to_native()));
115 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
116 }
117 }
118
120 template<isa<arm> Arch> requires(!Arch.has(arm_feature::i8mm) && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
124 simd<std::int8_t, 8, Arch> b) noexcept {
125 return detail::arm_constant::dot<false, -1>(acc, a, b);
126 }
127
129 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 2)
130 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
131 constexpr simd<std::int32_t, 2, Arch> usdot_lane(
132 simd<std::int32_t, 2, Arch> acc,
133 simd<std::uint8_t, 8, Arch> a,
134 simd<std::int8_t, 8, Arch> b) noexcept {
135 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
136 else {
137 auto result = detail::usdot_lane<Arch, Lane>(
138 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
139 __builtin_bit_cast(uint8x8_t, a.to_native()),
140 __builtin_bit_cast(int8x8_t, b.to_native()));
141 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
142 }
143 }
144
146 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 2 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
150 simd<std::int8_t, 8, Arch> b) noexcept {
151 return detail::arm_constant::dot<false, Lane>(acc, a, b);
152 }
153
155 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 2)
156 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
157 constexpr simd<std::int32_t, 2, Arch> sudot_lane(
158 simd<std::int32_t, 2, Arch> acc,
159 simd<std::int8_t, 8, Arch> a,
160 simd<std::uint8_t, 8, Arch> b) noexcept {
161 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
162 else {
163 auto result = detail::sudot_lane<Arch, Lane>(
164 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
165 __builtin_bit_cast(int8x8_t, a.to_native()),
166 __builtin_bit_cast(uint8x8_t, b.to_native()));
167 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
168 }
169 }
170
172 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 2 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::int8_t, 8, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); })
176 simd<std::uint8_t, 8, Arch> b) noexcept {
177 return detail::arm_constant::dot<false, Lane>(acc, a, b);
178 }
179
181 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 4)
182 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
183 constexpr simd<std::int32_t, 2, Arch> usdot_lane(
184 simd<std::int32_t, 2, Arch> acc,
185 simd<std::uint8_t, 8, Arch> a,
186 simd<std::int8_t, 16, Arch> b) noexcept {
187 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
188 else {
189 auto result = detail::usdot_lane<Arch, Lane>(
190 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
191 __builtin_bit_cast(uint8x8_t, a.to_native()),
192 __builtin_bit_cast(int8x16_t, b.to_native()));
193 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
194 }
195 }
196
198 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 4 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
202 simd<std::int8_t, 16, Arch> b) noexcept {
203 return detail::arm_constant::dot<false, Lane>(acc, a, b);
204 }
205
207 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 4)
208 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
209 constexpr simd<std::int32_t, 2, Arch> sudot_lane(
210 simd<std::int32_t, 2, Arch> acc,
211 simd<std::int8_t, 8, Arch> a,
212 simd<std::uint8_t, 16, Arch> b) noexcept {
213 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
214 else {
215 auto result = detail::sudot_lane<Arch, Lane>(
216 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
217 __builtin_bit_cast(int8x8_t, a.to_native()),
218 __builtin_bit_cast(uint8x16_t, b.to_native()));
219 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
220 }
221 }
222
224 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 4 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::int8_t, 8, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
228 simd<std::uint8_t, 16, Arch> b) noexcept {
229 return detail::arm_constant::dot<false, Lane>(acc, a, b);
230 }
231
233 template<isa<arm> Arch> requires(Arch.has(arm_feature::i8mm))
234 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
235 constexpr simd<std::int32_t, 4, Arch> usdot(
236 simd<std::int32_t, 4, Arch> acc,
237 simd<std::uint8_t, 16, Arch> a,
238 simd<std::int8_t, 16, Arch> b) noexcept {
239 if consteval { return detail::arm_constant::dot<false, -1>(acc, a, b); }
240 else {
241 auto result = detail::usdot<Arch>(
242 __builtin_bit_cast(int32x4_t, acc.to_native()),
243 __builtin_bit_cast(uint8x16_t, a.to_native()),
244 __builtin_bit_cast(int8x16_t, b.to_native()));
246 }
247 }
248
250 template<isa<arm> Arch> requires(!Arch.has(arm_feature::i8mm) && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
254 simd<std::int8_t, 16, Arch> b) noexcept {
255 return detail::arm_constant::dot<false, -1>(acc, a, b);
256 }
257
259 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 2)
260 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
261 constexpr simd<std::int32_t, 4, Arch> usdot_lane(
262 simd<std::int32_t, 4, Arch> acc,
263 simd<std::uint8_t, 16, Arch> a,
264 simd<std::int8_t, 8, Arch> b) noexcept {
265 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
266 else {
267 auto result = detail::usdot_lane<Arch, Lane>(
268 __builtin_bit_cast(int32x4_t, acc.to_native()),
269 __builtin_bit_cast(uint8x16_t, a.to_native()),
270 __builtin_bit_cast(int8x8_t, b.to_native()));
272 }
273 }
274
276 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 2 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
280 simd<std::int8_t, 8, Arch> b) noexcept {
281 return detail::arm_constant::dot<false, Lane>(acc, a, b);
282 }
283
285 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 2)
286 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
287 constexpr simd<std::int32_t, 4, Arch> sudot_lane(
288 simd<std::int32_t, 4, Arch> acc,
289 simd<std::int8_t, 16, Arch> a,
290 simd<std::uint8_t, 8, Arch> b) noexcept {
291 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
292 else {
293 auto result = detail::sudot_lane<Arch, Lane>(
294 __builtin_bit_cast(int32x4_t, acc.to_native()),
295 __builtin_bit_cast(int8x16_t, a.to_native()),
296 __builtin_bit_cast(uint8x8_t, b.to_native()));
298 }
299 }
300
302 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 2 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); })
306 simd<std::uint8_t, 8, Arch> b) noexcept {
307 return detail::arm_constant::dot<false, Lane>(acc, a, b);
308 }
309
311 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 4)
312 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
313 constexpr simd<std::int32_t, 4, Arch> usdot_lane(
314 simd<std::int32_t, 4, Arch> acc,
315 simd<std::uint8_t, 16, Arch> a,
316 simd<std::int8_t, 16, Arch> b) noexcept {
317 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
318 else {
319 auto result = detail::usdot_lane<Arch, Lane>(
320 __builtin_bit_cast(int32x4_t, acc.to_native()),
321 __builtin_bit_cast(uint8x16_t, a.to_native()),
322 __builtin_bit_cast(int8x16_t, b.to_native()));
324 }
325 }
326
328 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 4 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
332 simd<std::int8_t, 16, Arch> b) noexcept {
333 return detail::arm_constant::dot<false, Lane>(acc, a, b);
334 }
335
337 template<isa<arm> Arch, unsigned Lane> requires(Arch.has(arm_feature::i8mm) && Lane < 4)
338 native_nodiscard native_inline native_const __attribute__((target("i8mm")))
339 constexpr simd<std::int32_t, 4, Arch> sudot_lane(
340 simd<std::int32_t, 4, Arch> acc,
341 simd<std::int8_t, 16, Arch> a,
342 simd<std::uint8_t, 16, Arch> b) noexcept {
343 if consteval { return detail::arm_constant::dot<false, Lane>(acc, a, b); }
344 else {
345 auto result = detail::sudot_lane<Arch, Lane>(
346 __builtin_bit_cast(int32x4_t, acc.to_native()),
347 __builtin_bit_cast(int8x16_t, a.to_native()),
348 __builtin_bit_cast(uint8x16_t, b.to_native()));
350 }
351 }
352
354 template<isa<arm> Arch, unsigned Lane> requires(!Arch.has(arm_feature::i8mm) && Lane < 4 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
358 simd<std::uint8_t, 16, Arch> b) noexcept {
359 return detail::arm_constant::dot<false, Lane>(acc, a, b);
360 }
361
363 template<isa<arm> Arch, class A, class B, class C>
364 void smmla(A, B, C) = delete;
365 template<isa<arm> Arch, class A, class B, class C>
366 void ummla(A, B, C) = delete;
367 template<isa<arm> Arch, class A, class B, class C>
368 void usmmla(A, B, C) = delete;
369 template<isa<arm> Arch, class A, class B, class C>
370 void usdot(A, B, C) = delete;
371 template<isa<arm> Arch, unsigned Lane, class A, class B, class C>
372 void usdot_lane(A, B, C) = delete;
373 template<isa<arm> Arch, unsigned Lane, class A, class B, class C>
374 void sudot_lane(A, B, C) = delete;
377}
378#endif
379}
constexpr simd< std::int32_t, 2, Arch > usdot(simd< std::int32_t, 2, Arch > acc, simd< std::uint8_t, 8, Arch > a, simd< std::int8_t, 8, Arch > b) noexcept
Accumulate each corresponding group of four unsigned a bytes times signed b bytes.
constexpr simd< std::int32_t, 2, Arch > sudot_lane(simd< std::int32_t, 2, Arch > acc, simd< std::int8_t, 8, Arch > a, simd< std::uint8_t, 8, Arch > b) noexcept
Accumulate each four-byte group of a times b[4*Lane..4*Lane+3].
constexpr simd< std::int32_t, 4, Arch > usmmla(simd< std::int32_t, 4, Arch > acc, simd< std::uint8_t, 16, Arch > a, simd< std::int8_t, 16, Arch > b) noexcept
Unsigned left matrix times signed right matrix, with signed accumulator.
constexpr simd< std::uint32_t, 4, Arch > ummla(simd< std::uint32_t, 4, Arch > acc, simd< std::uint8_t, 16, Arch > a, simd< std::uint8_t, 16, Arch > b) noexcept
Unsigned 2x8 times 8x2 matrix multiply-accumulate.
constexpr simd< std::int32_t, 4, Arch > smmla(simd< std::int32_t, 4, Arch > acc, simd< std::int8_t, 16, Arch > a, simd< std::int8_t, 16, Arch > b) noexcept
Signed 2x8 times 8x2 matrix multiply-accumulate.
constexpr simd< std::int32_t, 2, Arch > usdot_lane(simd< std::int32_t, 2, Arch > acc, simd< std::uint8_t, 8, Arch > a, simd< std::int8_t, 8, Arch > b) noexcept
Accumulate each four-byte group of a times b[4*Lane..4*Lane+3].
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
#define native_const
[[const]] is not const
Definition attributes.h:108
Architecture-tagged vectors, register packs and supporting value types. Native arithmetic follows its...
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Definition isa.h:396
Standard-library adaptations documented here for SIMD value types.
Omitted architecture arguments use the native.simd provider's baseline.