3#include "native/isa_import.h"
4#include "native/arm/detail/integer_constant.h"
5#include "native/arm/i8mm.h"
10#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
25 template<isa<arm> Arch>
requires(Arch.has(arm_feature::i8mm))
28 simd<
std::int32_t, 4, Arch> acc,
30 simd<
std::int8_t, 16, Arch> b) noexcept {
31 if consteval {
return detail::arm_constant::dot<
true, -1>(acc, a, b); }
33 auto result = detail::smmla<Arch>(
34 __builtin_bit_cast(int32x4_t, acc.to_native()),
35 __builtin_bit_cast(int8x16_t, a.to_native()),
36 __builtin_bit_cast(int8x16_t, b.to_native()));
42 template<isa<arm> Arch>
requires(!Arch.has(arm_feature::i8mm) &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::int8_t, 16, Arch>); })
47 return detail::arm_constant::dot<
true, -1>(acc, a, b);
51 template<isa<arm> Arch>
requires(Arch.has(arm_feature::i8mm))
54 simd<
std::uint32_t, 4, Arch> acc,
56 simd<
std::uint8_t, 16, Arch> b) noexcept {
57 if consteval {
return detail::arm_constant::dot<
true, -1>(acc, a, b); }
59 auto result = detail::ummla<Arch>(
60 __builtin_bit_cast(uint32x4_t, acc.to_native()),
61 __builtin_bit_cast(uint8x16_t, a.to_native()),
62 __builtin_bit_cast(uint8x16_t, b.to_native()));
68 template<isa<arm> Arch>
requires(!Arch.has(arm_feature::i8mm) &&
requires {
sizeof(simd<std::uint32_t, 4, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>); })
73 return detail::arm_constant::dot<
true, -1>(acc, a, b);
77 template<isa<arm> Arch>
requires(Arch.has(arm_feature::i8mm))
80 simd<
std::int32_t, 4, Arch> acc,
82 simd<
std::int8_t, 16, Arch> b) noexcept {
83 if consteval {
return detail::arm_constant::dot<
true, -1>(acc, a, b); }
85 auto result = detail::usmmla<Arch>(
86 __builtin_bit_cast(int32x4_t, acc.to_native()),
87 __builtin_bit_cast(uint8x16_t, a.to_native()),
88 __builtin_bit_cast(int8x16_t, b.to_native()));
94 template<isa<arm> Arch>
requires(!Arch.has(arm_feature::i8mm) &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>);
sizeof(simd<std::int8_t, 16, Arch>); })
99 return detail::arm_constant::dot<
true, -1>(acc, a, b);
103 template<isa<arm> Arch>
requires(Arch.has(arm_feature::i8mm))
106 simd<
std::int32_t, 2, Arch> acc,
108 simd<
std::int8_t, 8, Arch> b) noexcept {
109 if consteval {
return detail::arm_constant::dot<
false, -1>(acc, a, b); }
111 auto result = detail::usdot<Arch>(
112 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
113 __builtin_bit_cast(uint8x8_t, a.to_native()),
114 __builtin_bit_cast(int8x8_t, b.to_native()));
120 template<isa<arm> Arch>
requires(!Arch.has(arm_feature::i8mm) &&
requires {
sizeof(simd<std::int32_t, 2, Arch>);
sizeof(simd<std::uint8_t, 8, Arch>);
sizeof(simd<std::int8_t, 8, Arch>); })
125 return detail::arm_constant::dot<
false, -1>(acc, a, b);
129 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 2)
132 simd<
std::int32_t, 2, Arch> acc,
134 simd<
std::int8_t, 8, Arch> b) noexcept {
135 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
137 auto result = detail::usdot_lane<Arch, Lane>(
138 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
139 __builtin_bit_cast(uint8x8_t, a.to_native()),
140 __builtin_bit_cast(int8x8_t, b.to_native()));
146 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 2 &&
requires {
sizeof(simd<std::int32_t, 2, Arch>);
sizeof(simd<std::uint8_t, 8, Arch>);
sizeof(simd<std::int8_t, 8, Arch>); })
151 return detail::arm_constant::dot<false, Lane>(acc, a, b);
155 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 2)
158 simd<
std::int32_t, 2, Arch> acc,
160 simd<
std::uint8_t, 8, Arch> b) noexcept {
161 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
163 auto result = detail::sudot_lane<Arch, Lane>(
164 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
165 __builtin_bit_cast(int8x8_t, a.to_native()),
166 __builtin_bit_cast(uint8x8_t, b.to_native()));
172 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 2 &&
requires {
sizeof(simd<std::int32_t, 2, Arch>);
sizeof(simd<std::int8_t, 8, Arch>);
sizeof(simd<std::uint8_t, 8, Arch>); })
177 return detail::arm_constant::dot<false, Lane>(acc, a, b);
181 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 4)
184 simd<
std::int32_t, 2, Arch> acc,
186 simd<
std::int8_t, 16, Arch> b) noexcept {
187 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
189 auto result = detail::usdot_lane<Arch, Lane>(
190 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
191 __builtin_bit_cast(uint8x8_t, a.to_native()),
192 __builtin_bit_cast(int8x16_t, b.to_native()));
198 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 4 &&
requires {
sizeof(simd<std::int32_t, 2, Arch>);
sizeof(simd<std::uint8_t, 8, Arch>);
sizeof(simd<std::int8_t, 16, Arch>); })
203 return detail::arm_constant::dot<false, Lane>(acc, a, b);
207 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 4)
210 simd<
std::int32_t, 2, Arch> acc,
212 simd<
std::uint8_t, 16, Arch> b) noexcept {
213 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
215 auto result = detail::sudot_lane<Arch, Lane>(
216 vget_low_s32(__builtin_bit_cast(int32x4_t, acc.to_native())),
217 __builtin_bit_cast(int8x8_t, a.to_native()),
218 __builtin_bit_cast(uint8x16_t, b.to_native()));
224 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 4 &&
requires {
sizeof(simd<std::int32_t, 2, Arch>);
sizeof(simd<std::int8_t, 8, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>); })
229 return detail::arm_constant::dot<false, Lane>(acc, a, b);
233 template<isa<arm> Arch>
requires(Arch.has(arm_feature::i8mm))
236 simd<
std::int32_t, 4, Arch> acc,
237 simd<
std::uint8_t, 16, Arch> a,
238 simd<
std::int8_t, 16, Arch> b) noexcept {
239 if consteval {
return detail::arm_constant::dot<
false, -1>(acc, a, b); }
241 auto result = detail::usdot<Arch>(
242 __builtin_bit_cast(int32x4_t, acc.to_native()),
243 __builtin_bit_cast(uint8x16_t, a.to_native()),
244 __builtin_bit_cast(int8x16_t, b.to_native()));
250 template<isa<arm> Arch>
requires(!Arch.has(arm_feature::i8mm) &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>);
sizeof(simd<std::int8_t, 16, Arch>); })
255 return detail::arm_constant::dot<
false, -1>(acc, a, b);
259 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 2)
262 simd<
std::int32_t, 4, Arch> acc,
263 simd<
std::uint8_t, 16, Arch> a,
264 simd<
std::int8_t, 8, Arch> b) noexcept {
265 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
267 auto result = detail::usdot_lane<Arch, Lane>(
268 __builtin_bit_cast(int32x4_t, acc.to_native()),
269 __builtin_bit_cast(uint8x16_t, a.to_native()),
270 __builtin_bit_cast(int8x8_t, b.to_native()));
276 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 2 &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>);
sizeof(simd<std::int8_t, 8, Arch>); })
281 return detail::arm_constant::dot<false, Lane>(acc, a, b);
285 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 2)
288 simd<
std::int32_t, 4, Arch> acc,
290 simd<
std::uint8_t, 8, Arch> b) noexcept {
291 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
293 auto result = detail::sudot_lane<Arch, Lane>(
294 __builtin_bit_cast(int32x4_t, acc.to_native()),
295 __builtin_bit_cast(int8x16_t, a.to_native()),
296 __builtin_bit_cast(uint8x8_t, b.to_native()));
302 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 2 &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::int8_t, 16, Arch>);
sizeof(simd<std::uint8_t, 8, Arch>); })
307 return detail::arm_constant::dot<false, Lane>(acc, a, b);
311 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 4)
314 simd<
std::int32_t, 4, Arch> acc,
315 simd<
std::uint8_t, 16, Arch> a,
316 simd<
std::int8_t, 16, Arch> b) noexcept {
317 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
319 auto result = detail::usdot_lane<Arch, Lane>(
320 __builtin_bit_cast(int32x4_t, acc.to_native()),
321 __builtin_bit_cast(uint8x16_t, a.to_native()),
322 __builtin_bit_cast(int8x16_t, b.to_native()));
328 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 4 &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>);
sizeof(simd<std::int8_t, 16, Arch>); })
333 return detail::arm_constant::dot<false, Lane>(acc, a, b);
337 template<isa<arm> Arch,
unsigned Lane>
requires(Arch.has(arm_feature::i8mm) && Lane < 4)
340 simd<
std::int32_t, 4, Arch> acc,
342 simd<
std::uint8_t, 16, Arch> b) noexcept {
343 if consteval {
return detail::arm_constant::dot<false, Lane>(acc, a, b); }
345 auto result = detail::sudot_lane<Arch, Lane>(
346 __builtin_bit_cast(int32x4_t, acc.to_native()),
347 __builtin_bit_cast(int8x16_t, a.to_native()),
348 __builtin_bit_cast(uint8x16_t, b.to_native()));
354 template<isa<arm> Arch,
unsigned Lane>
requires(!Arch.has(arm_feature::i8mm) && Lane < 4 &&
requires {
sizeof(simd<std::int32_t, 4, Arch>);
sizeof(simd<std::int8_t, 16, Arch>);
sizeof(simd<std::uint8_t, 16, Arch>); })
359 return detail::arm_constant::dot<false, Lane>(acc, a, b);
363 template<isa<arm> Arch,
class A,
class B,
class C>
364 void smmla(A, B, C) =
delete;
365 template<isa<arm> Arch,
class A,
class B,
class C>
366 void ummla(A, B, C) =
delete;
367 template<isa<arm> Arch,
class A,
class B,
class C>
368 void usmmla(A, B, C) =
delete;
369 template<isa<arm> Arch,
class A,
class B,
class C>
370 void usdot(A, B, C) =
delete;
371 template<isa<arm> Arch,
unsigned Lane,
class A,
class B,
class C>
372 void usdot_lane(A, B, C) =
delete;
373 template<isa<arm> Arch,
unsigned Lane,
class A,
class B,
class C>
374 void sudot_lane(A, B, C) =
delete;
constexpr simd< std::int32_t, 2, Arch > usdot(simd< std::int32_t, 2, Arch > acc, simd< std::uint8_t, 8, Arch > a, simd< std::int8_t, 8, Arch > b) noexcept
Accumulate each corresponding group of four unsigned a bytes times signed b bytes.
constexpr simd< std::int32_t, 2, Arch > sudot_lane(simd< std::int32_t, 2, Arch > acc, simd< std::int8_t, 8, Arch > a, simd< std::uint8_t, 8, Arch > b) noexcept
Accumulate each four-byte group of a times b[4*Lane..4*Lane+3].
constexpr simd< std::int32_t, 4, Arch > usmmla(simd< std::int32_t, 4, Arch > acc, simd< std::uint8_t, 16, Arch > a, simd< std::int8_t, 16, Arch > b) noexcept
Unsigned left matrix times signed right matrix, with signed accumulator.
constexpr simd< std::uint32_t, 4, Arch > ummla(simd< std::uint32_t, 4, Arch > acc, simd< std::uint8_t, 16, Arch > a, simd< std::uint8_t, 16, Arch > b) noexcept
Unsigned 2x8 times 8x2 matrix multiply-accumulate.
constexpr simd< std::int32_t, 4, Arch > smmla(simd< std::int32_t, 4, Arch > acc, simd< std::int8_t, 16, Arch > a, simd< std::int8_t, 16, Arch > b) noexcept
Signed 2x8 times 8x2 matrix multiply-accumulate.
constexpr simd< std::int32_t, 2, Arch > usdot_lane(simd< std::int32_t, 2, Arch > acc, simd< std::uint8_t, 8, Arch > a, simd< std::int8_t, 8, Arch > b) noexcept
Accumulate each four-byte group of a times b[4*Lane..4*Lane+3].
#define native_inline
inline [[always_inline]]
#define native_nodiscard
C++17 [[nodiscard]].
#define native_const
[[const]] is not const
Architecture-tagged vectors, register packs and supporting value types. Native arithmetic follows its...
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Standard-library adaptations documented here for SIMD value types.
Omitted architecture arguments use the native.simd provider's baseline.