native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
native.arm.dotprod.ccm
1// SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>
2// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
3module;
4#include "native/isa_import.h"
5#include "native/arm/detail/integer_constant.h"
6#include "native/arm/dotprod.h"
7export module native.arm.dotprod;
8export import native.arm.features;
9export import native.simd;
10export {
11#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
12namespace native {
21 // All vector operands share Arch; native registers remain implementation details.
23 template<isa<arm> Arch> requires(Arch.has(arm_feature::dotprod))
24 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
25 constexpr simd<std::int32_t, 2, Arch> sdot(
26 simd<std::int32_t, 2, Arch> accumulator,
27 simd<std::int8_t, 8, Arch> lhs,
28 simd<std::int8_t, 8, Arch> rhs) noexcept {
29 if consteval { return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs); }
30 else {
31 auto result = detail::sdot<Arch>(
32 vget_low_s32(__builtin_bit_cast(int32x4_t, accumulator.to_native())),
33 __builtin_bit_cast(int8x8_t, lhs.to_native()),
34 __builtin_bit_cast(int8x8_t, rhs.to_native()));
35 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
36 }
37 }
38
40 template<isa<arm> Arch> requires(!Arch.has(arm_feature::dotprod) && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
41 native_nodiscard consteval simd<std::int32_t, 2, Arch> sdot(
42 simd<std::int32_t, 2, Arch> accumulator,
43 simd<std::int8_t, 8, Arch> lhs,
44 simd<std::int8_t, 8, Arch> rhs) noexcept {
45 return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs);
46 }
47
49 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2)
50 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
51 constexpr simd<std::int32_t, 2, Arch> sdot_lane(
52 simd<std::int32_t, 2, Arch> accumulator,
53 simd<std::int8_t, 8, Arch> lhs,
54 simd<std::int8_t, 8, Arch> rhs) noexcept {
55 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
56 else {
57 auto result = detail::sdot_lane<Arch, Lane>(
58 vget_low_s32(__builtin_bit_cast(int32x4_t, accumulator.to_native())),
59 __builtin_bit_cast(int8x8_t, lhs.to_native()),
60 __builtin_bit_cast(int8x8_t, rhs.to_native()));
61 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
62 }
63 }
64
66 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
67 native_nodiscard consteval simd<std::int32_t, 2, Arch> sdot_lane(
68 simd<std::int32_t, 2, Arch> accumulator,
69 simd<std::int8_t, 8, Arch> lhs,
70 simd<std::int8_t, 8, Arch> rhs) noexcept {
71 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
72 }
73
75 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4)
76 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
77 constexpr simd<std::int32_t, 2, Arch> sdot_lane(
78 simd<std::int32_t, 2, Arch> accumulator,
79 simd<std::int8_t, 8, Arch> lhs,
80 simd<std::int8_t, 16, Arch> rhs) noexcept {
81 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
82 else {
83 auto result = detail::sdot_lane<Arch, Lane>(
84 vget_low_s32(__builtin_bit_cast(int32x4_t, accumulator.to_native())),
85 __builtin_bit_cast(int8x8_t, lhs.to_native()),
86 __builtin_bit_cast(int8x16_t, rhs.to_native()));
87 return simd<std::int32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 2, Arch>::native_type, vcombine_s32(result, vdup_n_s32(0))));
88 }
89 }
90
92 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4 && requires { sizeof(simd<std::int32_t, 2, Arch>); sizeof(simd<std::int8_t, 8, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
93 native_nodiscard consteval simd<std::int32_t, 2, Arch> sdot_lane(
94 simd<std::int32_t, 2, Arch> accumulator,
95 simd<std::int8_t, 8, Arch> lhs,
96 simd<std::int8_t, 16, Arch> rhs) noexcept {
97 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
98 }
99
101 template<isa<arm> Arch> requires(Arch.has(arm_feature::dotprod))
102 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
103 constexpr simd<std::int32_t, 4, Arch> sdot(
104 simd<std::int32_t, 4, Arch> accumulator,
105 simd<std::int8_t, 16, Arch> lhs,
106 simd<std::int8_t, 16, Arch> rhs) noexcept {
107 if consteval { return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs); }
108 else {
109 auto result = detail::sdot<Arch>(
110 __builtin_bit_cast(int32x4_t, accumulator.to_native()),
111 __builtin_bit_cast(int8x16_t, lhs.to_native()),
112 __builtin_bit_cast(int8x16_t, rhs.to_native()));
113 return simd<std::int32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 4, Arch>::native_type, result));
114 }
115 }
116
118 template<isa<arm> Arch> requires(!Arch.has(arm_feature::dotprod) && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
119 native_nodiscard consteval simd<std::int32_t, 4, Arch> sdot(
120 simd<std::int32_t, 4, Arch> accumulator,
121 simd<std::int8_t, 16, Arch> lhs,
122 simd<std::int8_t, 16, Arch> rhs) noexcept {
123 return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs);
124 }
125
127 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2)
128 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
129 constexpr simd<std::int32_t, 4, Arch> sdot_lane(
130 simd<std::int32_t, 4, Arch> accumulator,
131 simd<std::int8_t, 16, Arch> lhs,
132 simd<std::int8_t, 8, Arch> rhs) noexcept {
133 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
134 else {
135 auto result = detail::sdot_lane<Arch, Lane>(
136 __builtin_bit_cast(int32x4_t, accumulator.to_native()),
137 __builtin_bit_cast(int8x16_t, lhs.to_native()),
138 __builtin_bit_cast(int8x8_t, rhs.to_native()));
139 return simd<std::int32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 4, Arch>::native_type, result));
140 }
141 }
142
144 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); sizeof(simd<std::int8_t, 8, Arch>); })
145 native_nodiscard consteval simd<std::int32_t, 4, Arch> sdot_lane(
146 simd<std::int32_t, 4, Arch> accumulator,
147 simd<std::int8_t, 16, Arch> lhs,
148 simd<std::int8_t, 8, Arch> rhs) noexcept {
149 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
150 }
151
153 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4)
154 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
155 constexpr simd<std::int32_t, 4, Arch> sdot_lane(
156 simd<std::int32_t, 4, Arch> accumulator,
157 simd<std::int8_t, 16, Arch> lhs,
158 simd<std::int8_t, 16, Arch> rhs) noexcept {
159 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
160 else {
161 auto result = detail::sdot_lane<Arch, Lane>(
162 __builtin_bit_cast(int32x4_t, accumulator.to_native()),
163 __builtin_bit_cast(int8x16_t, lhs.to_native()),
164 __builtin_bit_cast(int8x16_t, rhs.to_native()));
165 return simd<std::int32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::int32_t, 4, Arch>::native_type, result));
166 }
167 }
168
170 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4 && requires { sizeof(simd<std::int32_t, 4, Arch>); sizeof(simd<std::int8_t, 16, Arch>); })
171 native_nodiscard consteval simd<std::int32_t, 4, Arch> sdot_lane(
172 simd<std::int32_t, 4, Arch> accumulator,
173 simd<std::int8_t, 16, Arch> lhs,
174 simd<std::int8_t, 16, Arch> rhs) noexcept {
175 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
176 }
177
179 template<isa<arm> Arch> requires(Arch.has(arm_feature::dotprod))
180 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
181 constexpr simd<std::uint32_t, 2, Arch> udot(
182 simd<std::uint32_t, 2, Arch> accumulator,
183 simd<std::uint8_t, 8, Arch> lhs,
184 simd<std::uint8_t, 8, Arch> rhs) noexcept {
185 if consteval { return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs); }
186 else {
187 auto result = detail::udot<Arch>(
188 vget_low_u32(__builtin_bit_cast(uint32x4_t, accumulator.to_native())),
189 __builtin_bit_cast(uint8x8_t, lhs.to_native()),
190 __builtin_bit_cast(uint8x8_t, rhs.to_native()));
191 return simd<std::uint32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 2, Arch>::native_type, vcombine_u32(result, vdup_n_u32(0))));
192 }
193 }
194
196 template<isa<arm> Arch> requires(!Arch.has(arm_feature::dotprod) && requires { sizeof(simd<std::uint32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); })
197 native_nodiscard consteval simd<std::uint32_t, 2, Arch> udot(
198 simd<std::uint32_t, 2, Arch> accumulator,
199 simd<std::uint8_t, 8, Arch> lhs,
200 simd<std::uint8_t, 8, Arch> rhs) noexcept {
201 return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs);
202 }
203
205 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2)
206 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
207 constexpr simd<std::uint32_t, 2, Arch> udot_lane(
208 simd<std::uint32_t, 2, Arch> accumulator,
209 simd<std::uint8_t, 8, Arch> lhs,
210 simd<std::uint8_t, 8, Arch> rhs) noexcept {
211 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
212 else {
213 auto result = detail::udot_lane<Arch, Lane>(
214 vget_low_u32(__builtin_bit_cast(uint32x4_t, accumulator.to_native())),
215 __builtin_bit_cast(uint8x8_t, lhs.to_native()),
216 __builtin_bit_cast(uint8x8_t, rhs.to_native()));
217 return simd<std::uint32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 2, Arch>::native_type, vcombine_u32(result, vdup_n_u32(0))));
218 }
219 }
220
222 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2 && requires { sizeof(simd<std::uint32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); })
223 native_nodiscard consteval simd<std::uint32_t, 2, Arch> udot_lane(
224 simd<std::uint32_t, 2, Arch> accumulator,
225 simd<std::uint8_t, 8, Arch> lhs,
226 simd<std::uint8_t, 8, Arch> rhs) noexcept {
227 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
228 }
229
231 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4)
232 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
233 constexpr simd<std::uint32_t, 2, Arch> udot_lane(
234 simd<std::uint32_t, 2, Arch> accumulator,
235 simd<std::uint8_t, 8, Arch> lhs,
236 simd<std::uint8_t, 16, Arch> rhs) noexcept {
237 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
238 else {
239 auto result = detail::udot_lane<Arch, Lane>(
240 vget_low_u32(__builtin_bit_cast(uint32x4_t, accumulator.to_native())),
241 __builtin_bit_cast(uint8x8_t, lhs.to_native()),
242 __builtin_bit_cast(uint8x16_t, rhs.to_native()));
243 return simd<std::uint32_t, 2, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 2, Arch>::native_type, vcombine_u32(result, vdup_n_u32(0))));
244 }
245 }
246
248 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4 && requires { sizeof(simd<std::uint32_t, 2, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
249 native_nodiscard consteval simd<std::uint32_t, 2, Arch> udot_lane(
250 simd<std::uint32_t, 2, Arch> accumulator,
251 simd<std::uint8_t, 8, Arch> lhs,
252 simd<std::uint8_t, 16, Arch> rhs) noexcept {
253 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
254 }
255
257 template<isa<arm> Arch> requires(Arch.has(arm_feature::dotprod))
258 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
259 constexpr simd<std::uint32_t, 4, Arch> udot(
260 simd<std::uint32_t, 4, Arch> accumulator,
261 simd<std::uint8_t, 16, Arch> lhs,
262 simd<std::uint8_t, 16, Arch> rhs) noexcept {
263 if consteval { return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs); }
264 else {
265 auto result = detail::udot<Arch>(
266 __builtin_bit_cast(uint32x4_t, accumulator.to_native()),
267 __builtin_bit_cast(uint8x16_t, lhs.to_native()),
268 __builtin_bit_cast(uint8x16_t, rhs.to_native()));
269 return simd<std::uint32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 4, Arch>::native_type, result));
270 }
271 }
272
274 template<isa<arm> Arch> requires(!Arch.has(arm_feature::dotprod) && requires { sizeof(simd<std::uint32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
275 native_nodiscard consteval simd<std::uint32_t, 4, Arch> udot(
276 simd<std::uint32_t, 4, Arch> accumulator,
277 simd<std::uint8_t, 16, Arch> lhs,
278 simd<std::uint8_t, 16, Arch> rhs) noexcept {
279 return detail::arm_constant::dot<false, -1>(accumulator, lhs, rhs);
280 }
281
283 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2)
284 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
285 constexpr simd<std::uint32_t, 4, Arch> udot_lane(
286 simd<std::uint32_t, 4, Arch> accumulator,
287 simd<std::uint8_t, 16, Arch> lhs,
288 simd<std::uint8_t, 8, Arch> rhs) noexcept {
289 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
290 else {
291 auto result = detail::udot_lane<Arch, Lane>(
292 __builtin_bit_cast(uint32x4_t, accumulator.to_native()),
293 __builtin_bit_cast(uint8x16_t, lhs.to_native()),
294 __builtin_bit_cast(uint8x8_t, rhs.to_native()));
295 return simd<std::uint32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 4, Arch>::native_type, result));
296 }
297 }
298
300 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 2 && requires { sizeof(simd<std::uint32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); sizeof(simd<std::uint8_t, 8, Arch>); })
301 native_nodiscard consteval simd<std::uint32_t, 4, Arch> udot_lane(
302 simd<std::uint32_t, 4, Arch> accumulator,
303 simd<std::uint8_t, 16, Arch> lhs,
304 simd<std::uint8_t, 8, Arch> rhs) noexcept {
305 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
306 }
307
309 template<isa<arm> Arch, int Lane> requires(Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4)
310 native_nodiscard native_inline native_const __attribute__((target("dotprod")))
311 constexpr simd<std::uint32_t, 4, Arch> udot_lane(
312 simd<std::uint32_t, 4, Arch> accumulator,
313 simd<std::uint8_t, 16, Arch> lhs,
314 simd<std::uint8_t, 16, Arch> rhs) noexcept {
315 if consteval { return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs); }
316 else {
317 auto result = detail::udot_lane<Arch, Lane>(
318 __builtin_bit_cast(uint32x4_t, accumulator.to_native()),
319 __builtin_bit_cast(uint8x16_t, lhs.to_native()),
320 __builtin_bit_cast(uint8x16_t, rhs.to_native()));
321 return simd<std::uint32_t, 4, Arch>::from_native(__builtin_bit_cast(typename simd<std::uint32_t, 4, Arch>::native_type, result));
322 }
323 }
324
326 template<isa<arm> Arch, int Lane> requires(!Arch.has(arm_feature::dotprod) && Lane >= 0 && Lane < 4 && requires { sizeof(simd<std::uint32_t, 4, Arch>); sizeof(simd<std::uint8_t, 16, Arch>); })
327 native_nodiscard consteval simd<std::uint32_t, 4, Arch> udot_lane(
328 simd<std::uint32_t, 4, Arch> accumulator,
329 simd<std::uint8_t, 16, Arch> lhs,
330 simd<std::uint8_t, 16, Arch> rhs) noexcept {
331 return detail::arm_constant::dot<false, Lane>(accumulator, lhs, rhs);
332 }
333
335 template<isa<arm> Arch, class A, class B, class C>
336 void sdot(A, B, C) = delete;
337 template<isa<arm> Arch, int Lane, class A, class B, class C>
338 void sdot_lane(A, B, C) = delete;
339 template<isa<arm> Arch, class A, class B, class C>
340 void udot(A, B, C) = delete;
341 template<isa<arm> Arch, int Lane, class A, class B, class C>
342 void udot_lane(A, B, C) = delete;
345}
346#endif
347}
native::udot_lane
constexpr simd< std::uint32_t, 2, Arch > udot_lane(simd< std::uint32_t, 2, Arch > accumulator, simd< std::uint8_t, 8, Arch > lhs, simd< std::uint8_t, 8, Arch > rhs) noexcept
UDOT by element: broadcast rhs byte group Lane to every accumulator lane.
Definition native.arm.dotprod.ccm:207
native::sdot
constexpr simd< std::int32_t, 2, Arch > sdot(simd< std::int32_t, 2, Arch > accumulator, simd< std::int8_t, 8, Arch > lhs, simd< std::int8_t, 8, Arch > rhs) noexcept
SDOT: four byte products per accumulator lane, modulo 2^32.
Definition native.arm.dotprod.ccm:25
native::sdot_lane
constexpr simd< std::int32_t, 2, Arch > sdot_lane(simd< std::int32_t, 2, Arch > accumulator, simd< std::int8_t, 8, Arch > lhs, simd< std::int8_t, 8, Arch > rhs) noexcept
SDOT by element: broadcast rhs byte group Lane to every accumulator lane.
Definition native.arm.dotprod.ccm:51
native::udot
constexpr simd< std::uint32_t, 2, Arch > udot(simd< std::uint32_t, 2, Arch > accumulator, simd< std::uint8_t, 8, Arch > lhs, simd< std::uint8_t, 8, Arch > rhs) noexcept
UDOT: four byte products per accumulator lane, modulo 2^32.
Definition native.arm.dotprod.ccm:181
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native_const
#define native_const
[[const]] is not const
Definition attributes.h:108
native.arm.dotprod
native.arm.features
native.simd
native
Architecture-tagged vectors, register packs and supporting value types. Native arithmetic follows its...
Definition native.arm.aes.ccm:10
native::target
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Definition isa.h:396
std
Standard-library adaptations documented here for SIMD value types.
native::simd
Omitted architecture arguments use the native.simd provider's baseline.
Definition native.simd.ccm:19