native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
vbmi.h
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2#pragma once
3#include "native/config.h"
4#include "native/attributes.h"
5#include "native/isa.h"
6#if NATIVE_HOST_X86
7#include <immintrin.h>
8#endif
9#if NATIVE_HOST_X86 || defined(NATIVE_DOXYGEN)
10namespace native::detail::x86_vbmi {
11 // Internal register helpers for native.x86.vbmi.
12
13 template<isa<x86> Arch>
14 requires(Arch.has(x86_feature::avx512f) &&
15 Arch.has(x86_feature::avx512bw) &&
16 Arch.has(x86_feature::avx512vbmi) &&
17 Arch.has(x86_feature::avx512vl))
18 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
19 __m128i vpermb(
20 __m128i indices,
21 __m128i value) noexcept {
22 return _mm_permutexvar_epi8(indices, value);
23 }
24
25 template<isa<x86> Arch>
26 requires(Arch.has(x86_feature::avx512f) &&
27 Arch.has(x86_feature::avx512bw) &&
28 Arch.has(x86_feature::avx512vbmi) &&
29 Arch.has(x86_feature::avx512vl))
30 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
31 __m128i mask_vpermb(
32 __m128i source,
33 __mmask16 mask,
34 __m128i indices,
35 __m128i value) noexcept {
36 return _mm_mask_permutexvar_epi8(source, mask, indices, value);
37 }
38
39 template<isa<x86> Arch>
40 requires(Arch.has(x86_feature::avx512f) &&
41 Arch.has(x86_feature::avx512bw) &&
42 Arch.has(x86_feature::avx512vbmi) &&
43 Arch.has(x86_feature::avx512vl))
44 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
45 __m128i maskz_vpermb(
46 __mmask16 mask,
47 __m128i indices,
48 __m128i value) noexcept {
49 return _mm_maskz_permutexvar_epi8(mask, indices, value);
50 }
51
52 template<isa<x86> Arch>
53 requires(Arch.has(x86_feature::avx512f) &&
54 Arch.has(x86_feature::avx512bw) &&
55 Arch.has(x86_feature::avx512vbmi) &&
56 Arch.has(x86_feature::avx512vl))
57 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
58 __m128i vpermt2b(
59 __m128i a,
60 __m128i indices,
61 __m128i b) noexcept {
62 return _mm_permutex2var_epi8(a, indices, b);
63 }
64
65 template<isa<x86> Arch>
66 requires(Arch.has(x86_feature::avx512f) &&
67 Arch.has(x86_feature::avx512bw) &&
68 Arch.has(x86_feature::avx512vbmi) &&
69 Arch.has(x86_feature::avx512vl))
70 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
71 __m128i mask_vpermt2b(
72 __m128i a,
73 __mmask16 mask,
74 __m128i indices,
75 __m128i b) noexcept {
76 return _mm_mask_permutex2var_epi8(a, mask, indices, b);
77 }
78
79 template<isa<x86> Arch>
80 requires(Arch.has(x86_feature::avx512f) &&
81 Arch.has(x86_feature::avx512bw) &&
82 Arch.has(x86_feature::avx512vbmi) &&
83 Arch.has(x86_feature::avx512vl))
84 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
85 __m128i maskz_vpermt2b(
86 __mmask16 mask,
87 __m128i a,
88 __m128i indices,
89 __m128i b) noexcept {
90 return _mm_maskz_permutex2var_epi8(mask, a, indices, b);
91 }
92
93 template<isa<x86> Arch>
94 requires(Arch.has(x86_feature::avx512f) &&
95 Arch.has(x86_feature::avx512bw) &&
96 Arch.has(x86_feature::avx512vbmi) &&
97 Arch.has(x86_feature::avx512vl))
98 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
99 __m128i vpermi2b(
100 __m128i indices,
101 __m128i a,
102 __m128i b) noexcept {
103 return _mm_permutex2var_epi8(a, indices, b);
104 }
105
106 template<isa<x86> Arch>
107 requires(Arch.has(x86_feature::avx512f) &&
108 Arch.has(x86_feature::avx512bw) &&
109 Arch.has(x86_feature::avx512vbmi) &&
110 Arch.has(x86_feature::avx512vl))
111 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
112 __m128i mask_vpermi2b(
113 __m128i indices,
114 __mmask16 mask,
115 __m128i a,
116 __m128i b) noexcept {
117 return _mm_mask2_permutex2var_epi8(a, indices, mask, b);
118 }
119
120 template<isa<x86> Arch>
121 requires(Arch.has(x86_feature::avx512f) &&
122 Arch.has(x86_feature::avx512bw) &&
123 Arch.has(x86_feature::avx512vbmi) &&
124 Arch.has(x86_feature::avx512vl))
125 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
126 __m128i maskz_vpermi2b(
127 __mmask16 mask,
128 __m128i indices,
129 __m128i a,
130 __m128i b) noexcept {
131 return _mm_maskz_permutex2var_epi8(mask, a, indices, b);
132 }
133
134 template<isa<x86> Arch>
135 requires(Arch.has(x86_feature::avx512f) &&
136 Arch.has(x86_feature::avx512bw) &&
137 Arch.has(x86_feature::avx512vbmi) &&
138 Arch.has(x86_feature::avx512vl))
139 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
140 __m128i vpmultishiftqb(
141 __m128i control,
142 __m128i value) noexcept {
143 return _mm_multishift_epi64_epi8(control, value);
144 }
145
146 template<isa<x86> Arch>
147 requires(Arch.has(x86_feature::avx512f) &&
148 Arch.has(x86_feature::avx512bw) &&
149 Arch.has(x86_feature::avx512vbmi) &&
150 Arch.has(x86_feature::avx512vl))
151 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
152 __m128i mask_vpmultishiftqb(
153 __m128i source,
154 __mmask16 mask,
155 __m128i control,
156 __m128i value) noexcept {
157 return _mm_mask_multishift_epi64_epi8(source, mask, control, value);
158 }
159
160 template<isa<x86> Arch>
161 requires(Arch.has(x86_feature::avx512f) &&
162 Arch.has(x86_feature::avx512bw) &&
163 Arch.has(x86_feature::avx512vbmi) &&
164 Arch.has(x86_feature::avx512vl))
165 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
166 __m128i maskz_vpmultishiftqb(
167 __mmask16 mask,
168 __m128i control,
169 __m128i value) noexcept {
170 return _mm_maskz_multishift_epi64_epi8(mask, control, value);
171 }
172
173 template<isa<x86> Arch>
174 requires(Arch.has(x86_feature::avx512f) &&
175 Arch.has(x86_feature::avx512bw) &&
176 Arch.has(x86_feature::avx512vbmi) &&
177 Arch.has(x86_feature::avx512vl))
178 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
179 __m256i vpermb(
180 __m256i indices,
181 __m256i value) noexcept {
182 return _mm256_permutexvar_epi8(indices, value);
183 }
184
185 template<isa<x86> Arch>
186 requires(Arch.has(x86_feature::avx512f) &&
187 Arch.has(x86_feature::avx512bw) &&
188 Arch.has(x86_feature::avx512vbmi) &&
189 Arch.has(x86_feature::avx512vl))
190 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
191 __m256i mask_vpermb(
192 __m256i source,
193 __mmask32 mask,
194 __m256i indices,
195 __m256i value) noexcept {
196 return _mm256_mask_permutexvar_epi8(source, mask, indices, value);
197 }
198
199 template<isa<x86> Arch>
200 requires(Arch.has(x86_feature::avx512f) &&
201 Arch.has(x86_feature::avx512bw) &&
202 Arch.has(x86_feature::avx512vbmi) &&
203 Arch.has(x86_feature::avx512vl))
204 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
205 __m256i maskz_vpermb(
206 __mmask32 mask,
207 __m256i indices,
208 __m256i value) noexcept {
209 return _mm256_maskz_permutexvar_epi8(mask, indices, value);
210 }
211
212 template<isa<x86> Arch>
213 requires(Arch.has(x86_feature::avx512f) &&
214 Arch.has(x86_feature::avx512bw) &&
215 Arch.has(x86_feature::avx512vbmi) &&
216 Arch.has(x86_feature::avx512vl))
217 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
218 __m256i vpermt2b(
219 __m256i a,
220 __m256i indices,
221 __m256i b) noexcept {
222 return _mm256_permutex2var_epi8(a, indices, b);
223 }
224
225 template<isa<x86> Arch>
226 requires(Arch.has(x86_feature::avx512f) &&
227 Arch.has(x86_feature::avx512bw) &&
228 Arch.has(x86_feature::avx512vbmi) &&
229 Arch.has(x86_feature::avx512vl))
230 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
231 __m256i mask_vpermt2b(
232 __m256i a,
233 __mmask32 mask,
234 __m256i indices,
235 __m256i b) noexcept {
236 return _mm256_mask_permutex2var_epi8(a, mask, indices, b);
237 }
238
239 template<isa<x86> Arch>
240 requires(Arch.has(x86_feature::avx512f) &&
241 Arch.has(x86_feature::avx512bw) &&
242 Arch.has(x86_feature::avx512vbmi) &&
243 Arch.has(x86_feature::avx512vl))
244 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
245 __m256i maskz_vpermt2b(
246 __mmask32 mask,
247 __m256i a,
248 __m256i indices,
249 __m256i b) noexcept {
250 return _mm256_maskz_permutex2var_epi8(mask, a, indices, b);
251 }
252
253 template<isa<x86> Arch>
254 requires(Arch.has(x86_feature::avx512f) &&
255 Arch.has(x86_feature::avx512bw) &&
256 Arch.has(x86_feature::avx512vbmi) &&
257 Arch.has(x86_feature::avx512vl))
258 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
259 __m256i vpermi2b(
260 __m256i indices,
261 __m256i a,
262 __m256i b) noexcept {
263 return _mm256_permutex2var_epi8(a, indices, b);
264 }
265
266 template<isa<x86> Arch>
267 requires(Arch.has(x86_feature::avx512f) &&
268 Arch.has(x86_feature::avx512bw) &&
269 Arch.has(x86_feature::avx512vbmi) &&
270 Arch.has(x86_feature::avx512vl))
271 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
272 __m256i mask_vpermi2b(
273 __m256i indices,
274 __mmask32 mask,
275 __m256i a,
276 __m256i b) noexcept {
277 return _mm256_mask2_permutex2var_epi8(a, indices, mask, b);
278 }
279
280 template<isa<x86> Arch>
281 requires(Arch.has(x86_feature::avx512f) &&
282 Arch.has(x86_feature::avx512bw) &&
283 Arch.has(x86_feature::avx512vbmi) &&
284 Arch.has(x86_feature::avx512vl))
285 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
286 __m256i maskz_vpermi2b(
287 __mmask32 mask,
288 __m256i indices,
289 __m256i a,
290 __m256i b) noexcept {
291 return _mm256_maskz_permutex2var_epi8(mask, a, indices, b);
292 }
293
294 template<isa<x86> Arch>
295 requires(Arch.has(x86_feature::avx512f) &&
296 Arch.has(x86_feature::avx512bw) &&
297 Arch.has(x86_feature::avx512vbmi) &&
298 Arch.has(x86_feature::avx512vl))
299 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
300 __m256i vpmultishiftqb(
301 __m256i control,
302 __m256i value) noexcept {
303 return _mm256_multishift_epi64_epi8(control, value);
304 }
305
306 template<isa<x86> Arch>
307 requires(Arch.has(x86_feature::avx512f) &&
308 Arch.has(x86_feature::avx512bw) &&
309 Arch.has(x86_feature::avx512vbmi) &&
310 Arch.has(x86_feature::avx512vl))
311 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
312 __m256i mask_vpmultishiftqb(
313 __m256i source,
314 __mmask32 mask,
315 __m256i control,
316 __m256i value) noexcept {
317 return _mm256_mask_multishift_epi64_epi8(source, mask, control, value);
318 }
319
320 template<isa<x86> Arch>
321 requires(Arch.has(x86_feature::avx512f) &&
322 Arch.has(x86_feature::avx512bw) &&
323 Arch.has(x86_feature::avx512vbmi) &&
324 Arch.has(x86_feature::avx512vl))
325 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi,avx512vl")
326 __m256i maskz_vpmultishiftqb(
327 __mmask32 mask,
328 __m256i control,
329 __m256i value) noexcept {
330 return _mm256_maskz_multishift_epi64_epi8(mask, control, value);
331 }
332
333 template<isa<x86> Arch>
334 requires(Arch.has(x86_feature::avx512f) &&
335 Arch.has(x86_feature::avx512bw) &&
336 Arch.has(x86_feature::avx512vbmi))
337 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
338 __m512i vpermb(
339 __m512i indices,
340 __m512i value) noexcept {
341 return _mm512_permutexvar_epi8(indices, value);
342 }
343
344 template<isa<x86> Arch>
345 requires(Arch.has(x86_feature::avx512f) &&
346 Arch.has(x86_feature::avx512bw) &&
347 Arch.has(x86_feature::avx512vbmi))
348 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
349 __m512i mask_vpermb(
350 __m512i source,
351 __mmask64 mask,
352 __m512i indices,
353 __m512i value) noexcept {
354 return _mm512_mask_permutexvar_epi8(source, mask, indices, value);
355 }
356
357 template<isa<x86> Arch>
358 requires(Arch.has(x86_feature::avx512f) &&
359 Arch.has(x86_feature::avx512bw) &&
360 Arch.has(x86_feature::avx512vbmi))
361 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
362 __m512i maskz_vpermb(
363 __mmask64 mask,
364 __m512i indices,
365 __m512i value) noexcept {
366 return _mm512_maskz_permutexvar_epi8(mask, indices, value);
367 }
368
369 template<isa<x86> Arch>
370 requires(Arch.has(x86_feature::avx512f) &&
371 Arch.has(x86_feature::avx512bw) &&
372 Arch.has(x86_feature::avx512vbmi))
373 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
374 __m512i vpermt2b(
375 __m512i a,
376 __m512i indices,
377 __m512i b) noexcept {
378 return _mm512_permutex2var_epi8(a, indices, b);
379 }
380
381 template<isa<x86> Arch>
382 requires(Arch.has(x86_feature::avx512f) &&
383 Arch.has(x86_feature::avx512bw) &&
384 Arch.has(x86_feature::avx512vbmi))
385 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
386 __m512i mask_vpermt2b(
387 __m512i a,
388 __mmask64 mask,
389 __m512i indices,
390 __m512i b) noexcept {
391 return _mm512_mask_permutex2var_epi8(a, mask, indices, b);
392 }
393
394 template<isa<x86> Arch>
395 requires(Arch.has(x86_feature::avx512f) &&
396 Arch.has(x86_feature::avx512bw) &&
397 Arch.has(x86_feature::avx512vbmi))
398 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
399 __m512i maskz_vpermt2b(
400 __mmask64 mask,
401 __m512i a,
402 __m512i indices,
403 __m512i b) noexcept {
404 return _mm512_maskz_permutex2var_epi8(mask, a, indices, b);
405 }
406
407 template<isa<x86> Arch>
408 requires(Arch.has(x86_feature::avx512f) &&
409 Arch.has(x86_feature::avx512bw) &&
410 Arch.has(x86_feature::avx512vbmi))
411 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
412 __m512i vpermi2b(
413 __m512i indices,
414 __m512i a,
415 __m512i b) noexcept {
416 return _mm512_permutex2var_epi8(a, indices, b);
417 }
418
419 template<isa<x86> Arch>
420 requires(Arch.has(x86_feature::avx512f) &&
421 Arch.has(x86_feature::avx512bw) &&
422 Arch.has(x86_feature::avx512vbmi))
423 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
424 __m512i mask_vpermi2b(
425 __m512i indices,
426 __mmask64 mask,
427 __m512i a,
428 __m512i b) noexcept {
429 return _mm512_mask2_permutex2var_epi8(a, indices, mask, b);
430 }
431
432 template<isa<x86> Arch>
433 requires(Arch.has(x86_feature::avx512f) &&
434 Arch.has(x86_feature::avx512bw) &&
435 Arch.has(x86_feature::avx512vbmi))
436 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
437 __m512i maskz_vpermi2b(
438 __mmask64 mask,
439 __m512i indices,
440 __m512i a,
441 __m512i b) noexcept {
442 return _mm512_maskz_permutex2var_epi8(mask, a, indices, b);
443 }
444
445 template<isa<x86> Arch>
446 requires(Arch.has(x86_feature::avx512f) &&
447 Arch.has(x86_feature::avx512bw) &&
448 Arch.has(x86_feature::avx512vbmi))
449 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
450 __m512i vpmultishiftqb(
451 __m512i control,
452 __m512i value) noexcept {
453 return _mm512_multishift_epi64_epi8(control, value);
454 }
455
456 template<isa<x86> Arch>
457 requires(Arch.has(x86_feature::avx512f) &&
458 Arch.has(x86_feature::avx512bw) &&
459 Arch.has(x86_feature::avx512vbmi))
460 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
461 __m512i mask_vpmultishiftqb(
462 __m512i source,
463 __mmask64 mask,
464 __m512i control,
465 __m512i value) noexcept {
466 return _mm512_mask_multishift_epi64_epi8(source, mask, control, value);
467 }
468
469 template<isa<x86> Arch>
470 requires(Arch.has(x86_feature::avx512f) &&
471 Arch.has(x86_feature::avx512bw) &&
472 Arch.has(x86_feature::avx512vbmi))
473 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi")
474 __m512i maskz_vpmultishiftqb(
475 __mmask64 mask,
476 __m512i control,
477 __m512i value) noexcept {
478 return _mm512_maskz_multishift_epi64_epi8(mask, control, value);
479 }
480}
481#endif
482
483#include <array>
484#include <bit>
485#include <cstddef>
486#include <cstdint>
487
488namespace native::detail::x86_vbmi_constant {
489 template<bool Two, class V>
490 constexpr V permute(V indices, V a, V b, V source, std::uint64_t mask) noexcept {
491 std::array<std::uint8_t, V::lanes> selectors{}, first{}, second{}, result{};
492 indices.store(selectors.data());
493 a.store(first.data());
494 b.store(second.data());
495 source.store(result.data());
496 for (std::size_t lane = 0; lane < V::lanes; ++lane) {
497 if ((mask >> lane) & 1) {
498 auto index = selectors[lane] & (V::lanes * (Two ? 2 : 1) - 1);
499 result[lane] = index < V::lanes ? first[index] : second[index - V::lanes];
500 }
501 }
502 return V::load(result.data());
503 }
504
505 template<class C, class V>
506 constexpr C multishift(C control, V value, C source, std::uint64_t mask) noexcept {
507 std::array<std::uint8_t, C::lanes> selectors{}, result{};
508 std::array<std::uint64_t, V::lanes> words{};
509 control.store(selectors.data());
510 value.store(words.data());
511 source.store(result.data());
512 for (std::size_t lane = 0; lane < C::lanes; ++lane) {
513 if ((mask >> lane) & 1) {
514 // Eight selected bits wrap within the corresponding qword.
515 result[lane] = static_cast<std::uint8_t>(std::rotr(words[lane / 8], selectors[lane] & 63));
516 }
517 }
518 return C::load(result.data());
519 }
520}
attributes.h
Compiler attributes for host code, with shader-safe shared modifiers.
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native_const
#define native_const
[[const]] is not const
Definition attributes.h:108
native_target
#define native_target(x)
this indicates a required feature set for the current multiversioned function.
Definition attributes.h:476
native::mask
typename mask_traits< std::remove_cvref_t< T > >::type mask
Definition mask_traits.h:22