native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
vbmi2.h
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2#pragma once
3#include "native/config.h"
4#include "native/attributes.h"
5#include "native/isa.h"
6#if NATIVE_HOST_X86
7#include <immintrin.h>
8#endif
9#if NATIVE_HOST_X86 || defined(NATIVE_DOXYGEN)
10namespace native::detail::x86_vbmi2 {
11 // Internal register and memory helpers for native.x86.vbmi2.
12
13 template<isa<x86> Arch>
14 requires(Arch.has(x86_feature::avx512f) &&
15 Arch.has(x86_feature::avx512bw) &&
16 Arch.has(x86_feature::avx512vbmi2) &&
17 Arch.has(x86_feature::avx512vl))
18 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
19 __m128i mask_vpcompressb(
20 __m128i source,
21 __mmask16 mask,
22 __m128i value) noexcept {
23 return _mm_mask_compress_epi8(source, mask, value);
24 }
25
26 template<isa<x86> Arch>
27 requires(Arch.has(x86_feature::avx512f) &&
28 Arch.has(x86_feature::avx512bw) &&
29 Arch.has(x86_feature::avx512vbmi2) &&
30 Arch.has(x86_feature::avx512vl))
31 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
32 __m128i maskz_vpcompressb(
33 __mmask16 mask,
34 __m128i value) noexcept {
35 return _mm_maskz_compress_epi8(mask, value);
36 }
37
38 template<isa<x86> Arch>
39 requires(Arch.has(x86_feature::avx512f) &&
40 Arch.has(x86_feature::avx512bw) &&
41 Arch.has(x86_feature::avx512vbmi2) &&
42 Arch.has(x86_feature::avx512vl))
43 native_inline native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
44 void mask_vpcompressb(
45 void * destination,
46 __mmask16 mask,
47 __m128i value) noexcept {
48 _mm_mask_compressstoreu_epi8(destination, mask, value);
49 }
50
51 template<isa<x86> Arch>
52 requires(Arch.has(x86_feature::avx512f) &&
53 Arch.has(x86_feature::avx512bw) &&
54 Arch.has(x86_feature::avx512vbmi2) &&
55 Arch.has(x86_feature::avx512vl))
56 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
57 __m128i mask_vpexpandb(
58 __m128i source,
59 __mmask16 mask,
60 __m128i value) noexcept {
61 return _mm_mask_expand_epi8(source, mask, value);
62 }
63
64 template<isa<x86> Arch>
65 requires(Arch.has(x86_feature::avx512f) &&
66 Arch.has(x86_feature::avx512bw) &&
67 Arch.has(x86_feature::avx512vbmi2) &&
68 Arch.has(x86_feature::avx512vl))
69 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
70 __m128i maskz_vpexpandb(
71 __mmask16 mask,
72 __m128i value) noexcept {
73 return _mm_maskz_expand_epi8(mask, value);
74 }
75
76 template<isa<x86> Arch>
77 requires(Arch.has(x86_feature::avx512f) &&
78 Arch.has(x86_feature::avx512bw) &&
79 Arch.has(x86_feature::avx512vbmi2) &&
80 Arch.has(x86_feature::avx512vl))
81 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
82 __m128i mask_vpexpandb(
83 __m128i source,
84 __mmask16 mask,
85 void const * memory) noexcept {
86 return _mm_mask_expandloadu_epi8(source, mask, memory);
87 }
88
89 template<isa<x86> Arch>
90 requires(Arch.has(x86_feature::avx512f) &&
91 Arch.has(x86_feature::avx512bw) &&
92 Arch.has(x86_feature::avx512vbmi2) &&
93 Arch.has(x86_feature::avx512vl))
94 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
95 __m128i maskz_vpexpandb_128(
96 __mmask16 mask,
97 void const * memory) noexcept {
98 return _mm_maskz_expandloadu_epi8(mask, memory);
99 }
100
101 template<isa<x86> Arch>
102 requires(Arch.has(x86_feature::avx512f) &&
103 Arch.has(x86_feature::avx512bw) &&
104 Arch.has(x86_feature::avx512vbmi2) &&
105 Arch.has(x86_feature::avx512vl))
106 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
107 __m128i mask_vpcompressw(
108 __m128i source,
109 __mmask8 mask,
110 __m128i value) noexcept {
111 return _mm_mask_compress_epi16(source, mask, value);
112 }
113
114 template<isa<x86> Arch>
115 requires(Arch.has(x86_feature::avx512f) &&
116 Arch.has(x86_feature::avx512bw) &&
117 Arch.has(x86_feature::avx512vbmi2) &&
118 Arch.has(x86_feature::avx512vl))
119 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
120 __m128i maskz_vpcompressw(
121 __mmask8 mask,
122 __m128i value) noexcept {
123 return _mm_maskz_compress_epi16(mask, value);
124 }
125
126 template<isa<x86> Arch>
127 requires(Arch.has(x86_feature::avx512f) &&
128 Arch.has(x86_feature::avx512bw) &&
129 Arch.has(x86_feature::avx512vbmi2) &&
130 Arch.has(x86_feature::avx512vl))
131 native_inline native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
132 void mask_vpcompressw(
133 void * destination,
134 __mmask8 mask,
135 __m128i value) noexcept {
136 _mm_mask_compressstoreu_epi16(destination, mask, value);
137 }
138
139 template<isa<x86> Arch>
140 requires(Arch.has(x86_feature::avx512f) &&
141 Arch.has(x86_feature::avx512bw) &&
142 Arch.has(x86_feature::avx512vbmi2) &&
143 Arch.has(x86_feature::avx512vl))
144 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
145 __m128i mask_vpexpandw(
146 __m128i source,
147 __mmask8 mask,
148 __m128i value) noexcept {
149 return _mm_mask_expand_epi16(source, mask, value);
150 }
151
152 template<isa<x86> Arch>
153 requires(Arch.has(x86_feature::avx512f) &&
154 Arch.has(x86_feature::avx512bw) &&
155 Arch.has(x86_feature::avx512vbmi2) &&
156 Arch.has(x86_feature::avx512vl))
157 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
158 __m128i maskz_vpexpandw(
159 __mmask8 mask,
160 __m128i value) noexcept {
161 return _mm_maskz_expand_epi16(mask, value);
162 }
163
164 template<isa<x86> Arch>
165 requires(Arch.has(x86_feature::avx512f) &&
166 Arch.has(x86_feature::avx512bw) &&
167 Arch.has(x86_feature::avx512vbmi2) &&
168 Arch.has(x86_feature::avx512vl))
169 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
170 __m128i mask_vpexpandw(
171 __m128i source,
172 __mmask8 mask,
173 void const * memory) noexcept {
174 return _mm_mask_expandloadu_epi16(source, mask, memory);
175 }
176
177 template<isa<x86> Arch>
178 requires(Arch.has(x86_feature::avx512f) &&
179 Arch.has(x86_feature::avx512bw) &&
180 Arch.has(x86_feature::avx512vbmi2) &&
181 Arch.has(x86_feature::avx512vl))
182 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
183 __m128i maskz_vpexpandw_128(
184 __mmask8 mask,
185 void const * memory) noexcept {
186 return _mm_maskz_expandloadu_epi16(mask, memory);
187 }
188
189 template<isa<x86> Arch, unsigned Imm8>
190 requires(Arch.has(x86_feature::avx512f) &&
191 Arch.has(x86_feature::avx512bw) &&
192 Arch.has(x86_feature::avx512vbmi2) &&
193 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
194 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
195 __m128i vpshldw(
196 __m128i a,
197 __m128i b) noexcept {
198 return _mm_shldi_epi16(a, b, Imm8);
199 }
200
201 template<isa<x86> Arch, unsigned Imm8>
202 requires(Arch.has(x86_feature::avx512f) &&
203 Arch.has(x86_feature::avx512bw) &&
204 Arch.has(x86_feature::avx512vbmi2) &&
205 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
206 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
207 __m128i mask_vpshldw(
208 __m128i source,
209 __mmask8 mask,
210 __m128i a,
211 __m128i b) noexcept {
212 return _mm_mask_shldi_epi16(source, mask, a, b, Imm8);
213 }
214
215 template<isa<x86> Arch, unsigned Imm8>
216 requires(Arch.has(x86_feature::avx512f) &&
217 Arch.has(x86_feature::avx512bw) &&
218 Arch.has(x86_feature::avx512vbmi2) &&
219 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
220 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
221 __m128i maskz_vpshldw(
222 __mmask8 mask,
223 __m128i a,
224 __m128i b) noexcept {
225 return _mm_maskz_shldi_epi16(mask, a, b, Imm8);
226 }
227
228 template<isa<x86> Arch>
229 requires(Arch.has(x86_feature::avx512f) &&
230 Arch.has(x86_feature::avx512bw) &&
231 Arch.has(x86_feature::avx512vbmi2) &&
232 Arch.has(x86_feature::avx512vl))
233 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
234 __m128i vpshldvw(
235 __m128i a,
236 __m128i b,
237 __m128i counts) noexcept {
238 return _mm_shldv_epi16(a, b, counts);
239 }
240
241 template<isa<x86> Arch>
242 requires(Arch.has(x86_feature::avx512f) &&
243 Arch.has(x86_feature::avx512bw) &&
244 Arch.has(x86_feature::avx512vbmi2) &&
245 Arch.has(x86_feature::avx512vl))
246 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
247 __m128i mask_vpshldvw(
248 __m128i a,
249 __mmask8 mask,
250 __m128i b,
251 __m128i counts) noexcept {
252 return _mm_mask_shldv_epi16(a, mask, b, counts);
253 }
254
255 template<isa<x86> Arch>
256 requires(Arch.has(x86_feature::avx512f) &&
257 Arch.has(x86_feature::avx512bw) &&
258 Arch.has(x86_feature::avx512vbmi2) &&
259 Arch.has(x86_feature::avx512vl))
260 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
261 __m128i maskz_vpshldvw(
262 __mmask8 mask,
263 __m128i a,
264 __m128i b,
265 __m128i counts) noexcept {
266 return _mm_maskz_shldv_epi16(mask, a, b, counts);
267 }
268
269 template<isa<x86> Arch, unsigned Imm8>
270 requires(Arch.has(x86_feature::avx512f) &&
271 Arch.has(x86_feature::avx512bw) &&
272 Arch.has(x86_feature::avx512vbmi2) &&
273 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
274 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
275 __m128i vpshrdw(
276 __m128i a,
277 __m128i b) noexcept {
278 return _mm_shrdi_epi16(a, b, Imm8);
279 }
280
281 template<isa<x86> Arch, unsigned Imm8>
282 requires(Arch.has(x86_feature::avx512f) &&
283 Arch.has(x86_feature::avx512bw) &&
284 Arch.has(x86_feature::avx512vbmi2) &&
285 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
286 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
287 __m128i mask_vpshrdw(
288 __m128i source,
289 __mmask8 mask,
290 __m128i a,
291 __m128i b) noexcept {
292 return _mm_mask_shrdi_epi16(source, mask, a, b, Imm8);
293 }
294
295 template<isa<x86> Arch, unsigned Imm8>
296 requires(Arch.has(x86_feature::avx512f) &&
297 Arch.has(x86_feature::avx512bw) &&
298 Arch.has(x86_feature::avx512vbmi2) &&
299 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
300 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
301 __m128i maskz_vpshrdw(
302 __mmask8 mask,
303 __m128i a,
304 __m128i b) noexcept {
305 return _mm_maskz_shrdi_epi16(mask, a, b, Imm8);
306 }
307
308 template<isa<x86> Arch>
309 requires(Arch.has(x86_feature::avx512f) &&
310 Arch.has(x86_feature::avx512bw) &&
311 Arch.has(x86_feature::avx512vbmi2) &&
312 Arch.has(x86_feature::avx512vl))
313 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
314 __m128i vpshrdvw(
315 __m128i a,
316 __m128i b,
317 __m128i counts) noexcept {
318 return _mm_shrdv_epi16(a, b, counts);
319 }
320
321 template<isa<x86> Arch>
322 requires(Arch.has(x86_feature::avx512f) &&
323 Arch.has(x86_feature::avx512bw) &&
324 Arch.has(x86_feature::avx512vbmi2) &&
325 Arch.has(x86_feature::avx512vl))
326 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
327 __m128i mask_vpshrdvw(
328 __m128i a,
329 __mmask8 mask,
330 __m128i b,
331 __m128i counts) noexcept {
332 return _mm_mask_shrdv_epi16(a, mask, b, counts);
333 }
334
335 template<isa<x86> Arch>
336 requires(Arch.has(x86_feature::avx512f) &&
337 Arch.has(x86_feature::avx512bw) &&
338 Arch.has(x86_feature::avx512vbmi2) &&
339 Arch.has(x86_feature::avx512vl))
340 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
341 __m128i maskz_vpshrdvw(
342 __mmask8 mask,
343 __m128i a,
344 __m128i b,
345 __m128i counts) noexcept {
346 return _mm_maskz_shrdv_epi16(mask, a, b, counts);
347 }
348
349 template<isa<x86> Arch, unsigned Imm8>
350 requires(Arch.has(x86_feature::avx512f) &&
351 Arch.has(x86_feature::avx512bw) &&
352 Arch.has(x86_feature::avx512vbmi2) &&
353 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
354 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
355 __m128i vpshldd(
356 __m128i a,
357 __m128i b) noexcept {
358 return _mm_shldi_epi32(a, b, Imm8);
359 }
360
361 template<isa<x86> Arch, unsigned Imm8>
362 requires(Arch.has(x86_feature::avx512f) &&
363 Arch.has(x86_feature::avx512bw) &&
364 Arch.has(x86_feature::avx512vbmi2) &&
365 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
366 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
367 __m128i mask_vpshldd(
368 __m128i source,
369 __mmask8 mask,
370 __m128i a,
371 __m128i b) noexcept {
372 return _mm_mask_shldi_epi32(source, mask, a, b, Imm8);
373 }
374
375 template<isa<x86> Arch, unsigned Imm8>
376 requires(Arch.has(x86_feature::avx512f) &&
377 Arch.has(x86_feature::avx512bw) &&
378 Arch.has(x86_feature::avx512vbmi2) &&
379 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
380 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
381 __m128i maskz_vpshldd(
382 __mmask8 mask,
383 __m128i a,
384 __m128i b) noexcept {
385 return _mm_maskz_shldi_epi32(mask, a, b, Imm8);
386 }
387
388 template<isa<x86> Arch>
389 requires(Arch.has(x86_feature::avx512f) &&
390 Arch.has(x86_feature::avx512bw) &&
391 Arch.has(x86_feature::avx512vbmi2) &&
392 Arch.has(x86_feature::avx512vl))
393 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
394 __m128i vpshldvd(
395 __m128i a,
396 __m128i b,
397 __m128i counts) noexcept {
398 return _mm_shldv_epi32(a, b, counts);
399 }
400
401 template<isa<x86> Arch>
402 requires(Arch.has(x86_feature::avx512f) &&
403 Arch.has(x86_feature::avx512bw) &&
404 Arch.has(x86_feature::avx512vbmi2) &&
405 Arch.has(x86_feature::avx512vl))
406 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
407 __m128i mask_vpshldvd(
408 __m128i a,
409 __mmask8 mask,
410 __m128i b,
411 __m128i counts) noexcept {
412 return _mm_mask_shldv_epi32(a, mask, b, counts);
413 }
414
415 template<isa<x86> Arch>
416 requires(Arch.has(x86_feature::avx512f) &&
417 Arch.has(x86_feature::avx512bw) &&
418 Arch.has(x86_feature::avx512vbmi2) &&
419 Arch.has(x86_feature::avx512vl))
420 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
421 __m128i maskz_vpshldvd(
422 __mmask8 mask,
423 __m128i a,
424 __m128i b,
425 __m128i counts) noexcept {
426 return _mm_maskz_shldv_epi32(mask, a, b, counts);
427 }
428
429 template<isa<x86> Arch, unsigned Imm8>
430 requires(Arch.has(x86_feature::avx512f) &&
431 Arch.has(x86_feature::avx512bw) &&
432 Arch.has(x86_feature::avx512vbmi2) &&
433 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
434 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
435 __m128i vpshrdd(
436 __m128i a,
437 __m128i b) noexcept {
438 return _mm_shrdi_epi32(a, b, Imm8);
439 }
440
441 template<isa<x86> Arch, unsigned Imm8>
442 requires(Arch.has(x86_feature::avx512f) &&
443 Arch.has(x86_feature::avx512bw) &&
444 Arch.has(x86_feature::avx512vbmi2) &&
445 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
446 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
447 __m128i mask_vpshrdd(
448 __m128i source,
449 __mmask8 mask,
450 __m128i a,
451 __m128i b) noexcept {
452 return _mm_mask_shrdi_epi32(source, mask, a, b, Imm8);
453 }
454
455 template<isa<x86> Arch, unsigned Imm8>
456 requires(Arch.has(x86_feature::avx512f) &&
457 Arch.has(x86_feature::avx512bw) &&
458 Arch.has(x86_feature::avx512vbmi2) &&
459 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
460 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
461 __m128i maskz_vpshrdd(
462 __mmask8 mask,
463 __m128i a,
464 __m128i b) noexcept {
465 return _mm_maskz_shrdi_epi32(mask, a, b, Imm8);
466 }
467
468 template<isa<x86> Arch>
469 requires(Arch.has(x86_feature::avx512f) &&
470 Arch.has(x86_feature::avx512bw) &&
471 Arch.has(x86_feature::avx512vbmi2) &&
472 Arch.has(x86_feature::avx512vl))
473 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
474 __m128i vpshrdvd(
475 __m128i a,
476 __m128i b,
477 __m128i counts) noexcept {
478 return _mm_shrdv_epi32(a, b, counts);
479 }
480
481 template<isa<x86> Arch>
482 requires(Arch.has(x86_feature::avx512f) &&
483 Arch.has(x86_feature::avx512bw) &&
484 Arch.has(x86_feature::avx512vbmi2) &&
485 Arch.has(x86_feature::avx512vl))
486 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
487 __m128i mask_vpshrdvd(
488 __m128i a,
489 __mmask8 mask,
490 __m128i b,
491 __m128i counts) noexcept {
492 return _mm_mask_shrdv_epi32(a, mask, b, counts);
493 }
494
495 template<isa<x86> Arch>
496 requires(Arch.has(x86_feature::avx512f) &&
497 Arch.has(x86_feature::avx512bw) &&
498 Arch.has(x86_feature::avx512vbmi2) &&
499 Arch.has(x86_feature::avx512vl))
500 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
501 __m128i maskz_vpshrdvd(
502 __mmask8 mask,
503 __m128i a,
504 __m128i b,
505 __m128i counts) noexcept {
506 return _mm_maskz_shrdv_epi32(mask, a, b, counts);
507 }
508
509 template<isa<x86> Arch, unsigned Imm8>
510 requires(Arch.has(x86_feature::avx512f) &&
511 Arch.has(x86_feature::avx512bw) &&
512 Arch.has(x86_feature::avx512vbmi2) &&
513 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
514 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
515 __m128i vpshldq(
516 __m128i a,
517 __m128i b) noexcept {
518 return _mm_shldi_epi64(a, b, Imm8);
519 }
520
521 template<isa<x86> Arch, unsigned Imm8>
522 requires(Arch.has(x86_feature::avx512f) &&
523 Arch.has(x86_feature::avx512bw) &&
524 Arch.has(x86_feature::avx512vbmi2) &&
525 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
526 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
527 __m128i mask_vpshldq(
528 __m128i source,
529 __mmask8 mask,
530 __m128i a,
531 __m128i b) noexcept {
532 return _mm_mask_shldi_epi64(source, mask, a, b, Imm8);
533 }
534
535 template<isa<x86> Arch, unsigned Imm8>
536 requires(Arch.has(x86_feature::avx512f) &&
537 Arch.has(x86_feature::avx512bw) &&
538 Arch.has(x86_feature::avx512vbmi2) &&
539 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
540 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
541 __m128i maskz_vpshldq(
542 __mmask8 mask,
543 __m128i a,
544 __m128i b) noexcept {
545 return _mm_maskz_shldi_epi64(mask, a, b, Imm8);
546 }
547
548 template<isa<x86> Arch>
549 requires(Arch.has(x86_feature::avx512f) &&
550 Arch.has(x86_feature::avx512bw) &&
551 Arch.has(x86_feature::avx512vbmi2) &&
552 Arch.has(x86_feature::avx512vl))
553 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
554 __m128i vpshldvq(
555 __m128i a,
556 __m128i b,
557 __m128i counts) noexcept {
558 return _mm_shldv_epi64(a, b, counts);
559 }
560
561 template<isa<x86> Arch>
562 requires(Arch.has(x86_feature::avx512f) &&
563 Arch.has(x86_feature::avx512bw) &&
564 Arch.has(x86_feature::avx512vbmi2) &&
565 Arch.has(x86_feature::avx512vl))
566 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
567 __m128i mask_vpshldvq(
568 __m128i a,
569 __mmask8 mask,
570 __m128i b,
571 __m128i counts) noexcept {
572 return _mm_mask_shldv_epi64(a, mask, b, counts);
573 }
574
575 template<isa<x86> Arch>
576 requires(Arch.has(x86_feature::avx512f) &&
577 Arch.has(x86_feature::avx512bw) &&
578 Arch.has(x86_feature::avx512vbmi2) &&
579 Arch.has(x86_feature::avx512vl))
580 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
581 __m128i maskz_vpshldvq(
582 __mmask8 mask,
583 __m128i a,
584 __m128i b,
585 __m128i counts) noexcept {
586 return _mm_maskz_shldv_epi64(mask, a, b, counts);
587 }
588
589 template<isa<x86> Arch, unsigned Imm8>
590 requires(Arch.has(x86_feature::avx512f) &&
591 Arch.has(x86_feature::avx512bw) &&
592 Arch.has(x86_feature::avx512vbmi2) &&
593 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
594 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
595 __m128i vpshrdq(
596 __m128i a,
597 __m128i b) noexcept {
598 return _mm_shrdi_epi64(a, b, Imm8);
599 }
600
601 template<isa<x86> Arch, unsigned Imm8>
602 requires(Arch.has(x86_feature::avx512f) &&
603 Arch.has(x86_feature::avx512bw) &&
604 Arch.has(x86_feature::avx512vbmi2) &&
605 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
606 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
607 __m128i mask_vpshrdq(
608 __m128i source,
609 __mmask8 mask,
610 __m128i a,
611 __m128i b) noexcept {
612 return _mm_mask_shrdi_epi64(source, mask, a, b, Imm8);
613 }
614
615 template<isa<x86> Arch, unsigned Imm8>
616 requires(Arch.has(x86_feature::avx512f) &&
617 Arch.has(x86_feature::avx512bw) &&
618 Arch.has(x86_feature::avx512vbmi2) &&
619 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
620 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
621 __m128i maskz_vpshrdq(
622 __mmask8 mask,
623 __m128i a,
624 __m128i b) noexcept {
625 return _mm_maskz_shrdi_epi64(mask, a, b, Imm8);
626 }
627
628 template<isa<x86> Arch>
629 requires(Arch.has(x86_feature::avx512f) &&
630 Arch.has(x86_feature::avx512bw) &&
631 Arch.has(x86_feature::avx512vbmi2) &&
632 Arch.has(x86_feature::avx512vl))
633 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
634 __m128i vpshrdvq(
635 __m128i a,
636 __m128i b,
637 __m128i counts) noexcept {
638 return _mm_shrdv_epi64(a, b, counts);
639 }
640
641 template<isa<x86> Arch>
642 requires(Arch.has(x86_feature::avx512f) &&
643 Arch.has(x86_feature::avx512bw) &&
644 Arch.has(x86_feature::avx512vbmi2) &&
645 Arch.has(x86_feature::avx512vl))
646 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
647 __m128i mask_vpshrdvq(
648 __m128i a,
649 __mmask8 mask,
650 __m128i b,
651 __m128i counts) noexcept {
652 return _mm_mask_shrdv_epi64(a, mask, b, counts);
653 }
654
655 template<isa<x86> Arch>
656 requires(Arch.has(x86_feature::avx512f) &&
657 Arch.has(x86_feature::avx512bw) &&
658 Arch.has(x86_feature::avx512vbmi2) &&
659 Arch.has(x86_feature::avx512vl))
660 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
661 __m128i maskz_vpshrdvq(
662 __mmask8 mask,
663 __m128i a,
664 __m128i b,
665 __m128i counts) noexcept {
666 return _mm_maskz_shrdv_epi64(mask, a, b, counts);
667 }
668
669 template<isa<x86> Arch>
670 requires(Arch.has(x86_feature::avx512f) &&
671 Arch.has(x86_feature::avx512bw) &&
672 Arch.has(x86_feature::avx512vbmi2) &&
673 Arch.has(x86_feature::avx512vl))
674 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
675 __m256i mask_vpcompressb(
676 __m256i source,
677 __mmask32 mask,
678 __m256i value) noexcept {
679 return _mm256_mask_compress_epi8(source, mask, value);
680 }
681
682 template<isa<x86> Arch>
683 requires(Arch.has(x86_feature::avx512f) &&
684 Arch.has(x86_feature::avx512bw) &&
685 Arch.has(x86_feature::avx512vbmi2) &&
686 Arch.has(x86_feature::avx512vl))
687 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
688 __m256i maskz_vpcompressb(
689 __mmask32 mask,
690 __m256i value) noexcept {
691 return _mm256_maskz_compress_epi8(mask, value);
692 }
693
694 template<isa<x86> Arch>
695 requires(Arch.has(x86_feature::avx512f) &&
696 Arch.has(x86_feature::avx512bw) &&
697 Arch.has(x86_feature::avx512vbmi2) &&
698 Arch.has(x86_feature::avx512vl))
699 native_inline native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
700 void mask_vpcompressb(
701 void * destination,
702 __mmask32 mask,
703 __m256i value) noexcept {
704 _mm256_mask_compressstoreu_epi8(destination, mask, value);
705 }
706
707 template<isa<x86> Arch>
708 requires(Arch.has(x86_feature::avx512f) &&
709 Arch.has(x86_feature::avx512bw) &&
710 Arch.has(x86_feature::avx512vbmi2) &&
711 Arch.has(x86_feature::avx512vl))
712 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
713 __m256i mask_vpexpandb(
714 __m256i source,
715 __mmask32 mask,
716 __m256i value) noexcept {
717 return _mm256_mask_expand_epi8(source, mask, value);
718 }
719
720 template<isa<x86> Arch>
721 requires(Arch.has(x86_feature::avx512f) &&
722 Arch.has(x86_feature::avx512bw) &&
723 Arch.has(x86_feature::avx512vbmi2) &&
724 Arch.has(x86_feature::avx512vl))
725 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
726 __m256i maskz_vpexpandb(
727 __mmask32 mask,
728 __m256i value) noexcept {
729 return _mm256_maskz_expand_epi8(mask, value);
730 }
731
732 template<isa<x86> Arch>
733 requires(Arch.has(x86_feature::avx512f) &&
734 Arch.has(x86_feature::avx512bw) &&
735 Arch.has(x86_feature::avx512vbmi2) &&
736 Arch.has(x86_feature::avx512vl))
737 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
738 __m256i mask_vpexpandb(
739 __m256i source,
740 __mmask32 mask,
741 void const * memory) noexcept {
742 return _mm256_mask_expandloadu_epi8(source, mask, memory);
743 }
744
745 template<isa<x86> Arch>
746 requires(Arch.has(x86_feature::avx512f) &&
747 Arch.has(x86_feature::avx512bw) &&
748 Arch.has(x86_feature::avx512vbmi2) &&
749 Arch.has(x86_feature::avx512vl))
750 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
751 __m256i maskz_vpexpandb_256(
752 __mmask32 mask,
753 void const * memory) noexcept {
754 return _mm256_maskz_expandloadu_epi8(mask, memory);
755 }
756
757 template<isa<x86> Arch>
758 requires(Arch.has(x86_feature::avx512f) &&
759 Arch.has(x86_feature::avx512bw) &&
760 Arch.has(x86_feature::avx512vbmi2) &&
761 Arch.has(x86_feature::avx512vl))
762 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
763 __m256i mask_vpcompressw(
764 __m256i source,
765 __mmask16 mask,
766 __m256i value) noexcept {
767 return _mm256_mask_compress_epi16(source, mask, value);
768 }
769
770 template<isa<x86> Arch>
771 requires(Arch.has(x86_feature::avx512f) &&
772 Arch.has(x86_feature::avx512bw) &&
773 Arch.has(x86_feature::avx512vbmi2) &&
774 Arch.has(x86_feature::avx512vl))
775 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
776 __m256i maskz_vpcompressw(
777 __mmask16 mask,
778 __m256i value) noexcept {
779 return _mm256_maskz_compress_epi16(mask, value);
780 }
781
782 template<isa<x86> Arch>
783 requires(Arch.has(x86_feature::avx512f) &&
784 Arch.has(x86_feature::avx512bw) &&
785 Arch.has(x86_feature::avx512vbmi2) &&
786 Arch.has(x86_feature::avx512vl))
787 native_inline native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
788 void mask_vpcompressw(
789 void * destination,
790 __mmask16 mask,
791 __m256i value) noexcept {
792 _mm256_mask_compressstoreu_epi16(destination, mask, value);
793 }
794
795 template<isa<x86> Arch>
796 requires(Arch.has(x86_feature::avx512f) &&
797 Arch.has(x86_feature::avx512bw) &&
798 Arch.has(x86_feature::avx512vbmi2) &&
799 Arch.has(x86_feature::avx512vl))
800 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
801 __m256i mask_vpexpandw(
802 __m256i source,
803 __mmask16 mask,
804 __m256i value) noexcept {
805 return _mm256_mask_expand_epi16(source, mask, value);
806 }
807
808 template<isa<x86> Arch>
809 requires(Arch.has(x86_feature::avx512f) &&
810 Arch.has(x86_feature::avx512bw) &&
811 Arch.has(x86_feature::avx512vbmi2) &&
812 Arch.has(x86_feature::avx512vl))
813 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
814 __m256i maskz_vpexpandw(
815 __mmask16 mask,
816 __m256i value) noexcept {
817 return _mm256_maskz_expand_epi16(mask, value);
818 }
819
820 template<isa<x86> Arch>
821 requires(Arch.has(x86_feature::avx512f) &&
822 Arch.has(x86_feature::avx512bw) &&
823 Arch.has(x86_feature::avx512vbmi2) &&
824 Arch.has(x86_feature::avx512vl))
825 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
826 __m256i mask_vpexpandw(
827 __m256i source,
828 __mmask16 mask,
829 void const * memory) noexcept {
830 return _mm256_mask_expandloadu_epi16(source, mask, memory);
831 }
832
833 template<isa<x86> Arch>
834 requires(Arch.has(x86_feature::avx512f) &&
835 Arch.has(x86_feature::avx512bw) &&
836 Arch.has(x86_feature::avx512vbmi2) &&
837 Arch.has(x86_feature::avx512vl))
838 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
839 __m256i maskz_vpexpandw_256(
840 __mmask16 mask,
841 void const * memory) noexcept {
842 return _mm256_maskz_expandloadu_epi16(mask, memory);
843 }
844
845 template<isa<x86> Arch, unsigned Imm8>
846 requires(Arch.has(x86_feature::avx512f) &&
847 Arch.has(x86_feature::avx512bw) &&
848 Arch.has(x86_feature::avx512vbmi2) &&
849 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
850 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
851 __m256i vpshldw(
852 __m256i a,
853 __m256i b) noexcept {
854 return _mm256_shldi_epi16(a, b, Imm8);
855 }
856
857 template<isa<x86> Arch, unsigned Imm8>
858 requires(Arch.has(x86_feature::avx512f) &&
859 Arch.has(x86_feature::avx512bw) &&
860 Arch.has(x86_feature::avx512vbmi2) &&
861 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
862 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
863 __m256i mask_vpshldw(
864 __m256i source,
865 __mmask16 mask,
866 __m256i a,
867 __m256i b) noexcept {
868 return _mm256_mask_shldi_epi16(source, mask, a, b, Imm8);
869 }
870
871 template<isa<x86> Arch, unsigned Imm8>
872 requires(Arch.has(x86_feature::avx512f) &&
873 Arch.has(x86_feature::avx512bw) &&
874 Arch.has(x86_feature::avx512vbmi2) &&
875 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
876 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
877 __m256i maskz_vpshldw(
878 __mmask16 mask,
879 __m256i a,
880 __m256i b) noexcept {
881 return _mm256_maskz_shldi_epi16(mask, a, b, Imm8);
882 }
883
884 template<isa<x86> Arch>
885 requires(Arch.has(x86_feature::avx512f) &&
886 Arch.has(x86_feature::avx512bw) &&
887 Arch.has(x86_feature::avx512vbmi2) &&
888 Arch.has(x86_feature::avx512vl))
889 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
890 __m256i vpshldvw(
891 __m256i a,
892 __m256i b,
893 __m256i counts) noexcept {
894 return _mm256_shldv_epi16(a, b, counts);
895 }
896
897 template<isa<x86> Arch>
898 requires(Arch.has(x86_feature::avx512f) &&
899 Arch.has(x86_feature::avx512bw) &&
900 Arch.has(x86_feature::avx512vbmi2) &&
901 Arch.has(x86_feature::avx512vl))
902 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
903 __m256i mask_vpshldvw(
904 __m256i a,
905 __mmask16 mask,
906 __m256i b,
907 __m256i counts) noexcept {
908 return _mm256_mask_shldv_epi16(a, mask, b, counts);
909 }
910
911 template<isa<x86> Arch>
912 requires(Arch.has(x86_feature::avx512f) &&
913 Arch.has(x86_feature::avx512bw) &&
914 Arch.has(x86_feature::avx512vbmi2) &&
915 Arch.has(x86_feature::avx512vl))
916 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
917 __m256i maskz_vpshldvw(
918 __mmask16 mask,
919 __m256i a,
920 __m256i b,
921 __m256i counts) noexcept {
922 return _mm256_maskz_shldv_epi16(mask, a, b, counts);
923 }
924
925 template<isa<x86> Arch, unsigned Imm8>
926 requires(Arch.has(x86_feature::avx512f) &&
927 Arch.has(x86_feature::avx512bw) &&
928 Arch.has(x86_feature::avx512vbmi2) &&
929 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
930 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
931 __m256i vpshrdw(
932 __m256i a,
933 __m256i b) noexcept {
934 return _mm256_shrdi_epi16(a, b, Imm8);
935 }
936
937 template<isa<x86> Arch, unsigned Imm8>
938 requires(Arch.has(x86_feature::avx512f) &&
939 Arch.has(x86_feature::avx512bw) &&
940 Arch.has(x86_feature::avx512vbmi2) &&
941 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
942 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
943 __m256i mask_vpshrdw(
944 __m256i source,
945 __mmask16 mask,
946 __m256i a,
947 __m256i b) noexcept {
948 return _mm256_mask_shrdi_epi16(source, mask, a, b, Imm8);
949 }
950
951 template<isa<x86> Arch, unsigned Imm8>
952 requires(Arch.has(x86_feature::avx512f) &&
953 Arch.has(x86_feature::avx512bw) &&
954 Arch.has(x86_feature::avx512vbmi2) &&
955 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
956 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
957 __m256i maskz_vpshrdw(
958 __mmask16 mask,
959 __m256i a,
960 __m256i b) noexcept {
961 return _mm256_maskz_shrdi_epi16(mask, a, b, Imm8);
962 }
963
964 template<isa<x86> Arch>
965 requires(Arch.has(x86_feature::avx512f) &&
966 Arch.has(x86_feature::avx512bw) &&
967 Arch.has(x86_feature::avx512vbmi2) &&
968 Arch.has(x86_feature::avx512vl))
969 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
970 __m256i vpshrdvw(
971 __m256i a,
972 __m256i b,
973 __m256i counts) noexcept {
974 return _mm256_shrdv_epi16(a, b, counts);
975 }
976
977 template<isa<x86> Arch>
978 requires(Arch.has(x86_feature::avx512f) &&
979 Arch.has(x86_feature::avx512bw) &&
980 Arch.has(x86_feature::avx512vbmi2) &&
981 Arch.has(x86_feature::avx512vl))
982 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
983 __m256i mask_vpshrdvw(
984 __m256i a,
985 __mmask16 mask,
986 __m256i b,
987 __m256i counts) noexcept {
988 return _mm256_mask_shrdv_epi16(a, mask, b, counts);
989 }
990
991 template<isa<x86> Arch>
992 requires(Arch.has(x86_feature::avx512f) &&
993 Arch.has(x86_feature::avx512bw) &&
994 Arch.has(x86_feature::avx512vbmi2) &&
995 Arch.has(x86_feature::avx512vl))
996 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
997 __m256i maskz_vpshrdvw(
998 __mmask16 mask,
999 __m256i a,
1000 __m256i b,
1001 __m256i counts) noexcept {
1002 return _mm256_maskz_shrdv_epi16(mask, a, b, counts);
1003 }
1004
1005 template<isa<x86> Arch, unsigned Imm8>
1006 requires(Arch.has(x86_feature::avx512f) &&
1007 Arch.has(x86_feature::avx512bw) &&
1008 Arch.has(x86_feature::avx512vbmi2) &&
1009 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1010 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1011 __m256i vpshldd(
1012 __m256i a,
1013 __m256i b) noexcept {
1014 return _mm256_shldi_epi32(a, b, Imm8);
1015 }
1016
1017 template<isa<x86> Arch, unsigned Imm8>
1018 requires(Arch.has(x86_feature::avx512f) &&
1019 Arch.has(x86_feature::avx512bw) &&
1020 Arch.has(x86_feature::avx512vbmi2) &&
1021 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1022 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1023 __m256i mask_vpshldd(
1024 __m256i source,
1025 __mmask8 mask,
1026 __m256i a,
1027 __m256i b) noexcept {
1028 return _mm256_mask_shldi_epi32(source, mask, a, b, Imm8);
1029 }
1030
1031 template<isa<x86> Arch, unsigned Imm8>
1032 requires(Arch.has(x86_feature::avx512f) &&
1033 Arch.has(x86_feature::avx512bw) &&
1034 Arch.has(x86_feature::avx512vbmi2) &&
1035 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1036 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1037 __m256i maskz_vpshldd(
1038 __mmask8 mask,
1039 __m256i a,
1040 __m256i b) noexcept {
1041 return _mm256_maskz_shldi_epi32(mask, a, b, Imm8);
1042 }
1043
1044 template<isa<x86> Arch>
1045 requires(Arch.has(x86_feature::avx512f) &&
1046 Arch.has(x86_feature::avx512bw) &&
1047 Arch.has(x86_feature::avx512vbmi2) &&
1048 Arch.has(x86_feature::avx512vl))
1049 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1050 __m256i vpshldvd(
1051 __m256i a,
1052 __m256i b,
1053 __m256i counts) noexcept {
1054 return _mm256_shldv_epi32(a, b, counts);
1055 }
1056
1057 template<isa<x86> Arch>
1058 requires(Arch.has(x86_feature::avx512f) &&
1059 Arch.has(x86_feature::avx512bw) &&
1060 Arch.has(x86_feature::avx512vbmi2) &&
1061 Arch.has(x86_feature::avx512vl))
1062 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1063 __m256i mask_vpshldvd(
1064 __m256i a,
1065 __mmask8 mask,
1066 __m256i b,
1067 __m256i counts) noexcept {
1068 return _mm256_mask_shldv_epi32(a, mask, b, counts);
1069 }
1070
1071 template<isa<x86> Arch>
1072 requires(Arch.has(x86_feature::avx512f) &&
1073 Arch.has(x86_feature::avx512bw) &&
1074 Arch.has(x86_feature::avx512vbmi2) &&
1075 Arch.has(x86_feature::avx512vl))
1076 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1077 __m256i maskz_vpshldvd(
1078 __mmask8 mask,
1079 __m256i a,
1080 __m256i b,
1081 __m256i counts) noexcept {
1082 return _mm256_maskz_shldv_epi32(mask, a, b, counts);
1083 }
1084
1085 template<isa<x86> Arch, unsigned Imm8>
1086 requires(Arch.has(x86_feature::avx512f) &&
1087 Arch.has(x86_feature::avx512bw) &&
1088 Arch.has(x86_feature::avx512vbmi2) &&
1089 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1090 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1091 __m256i vpshrdd(
1092 __m256i a,
1093 __m256i b) noexcept {
1094 return _mm256_shrdi_epi32(a, b, Imm8);
1095 }
1096
1097 template<isa<x86> Arch, unsigned Imm8>
1098 requires(Arch.has(x86_feature::avx512f) &&
1099 Arch.has(x86_feature::avx512bw) &&
1100 Arch.has(x86_feature::avx512vbmi2) &&
1101 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1102 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1103 __m256i mask_vpshrdd(
1104 __m256i source,
1105 __mmask8 mask,
1106 __m256i a,
1107 __m256i b) noexcept {
1108 return _mm256_mask_shrdi_epi32(source, mask, a, b, Imm8);
1109 }
1110
1111 template<isa<x86> Arch, unsigned Imm8>
1112 requires(Arch.has(x86_feature::avx512f) &&
1113 Arch.has(x86_feature::avx512bw) &&
1114 Arch.has(x86_feature::avx512vbmi2) &&
1115 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1116 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1117 __m256i maskz_vpshrdd(
1118 __mmask8 mask,
1119 __m256i a,
1120 __m256i b) noexcept {
1121 return _mm256_maskz_shrdi_epi32(mask, a, b, Imm8);
1122 }
1123
1124 template<isa<x86> Arch>
1125 requires(Arch.has(x86_feature::avx512f) &&
1126 Arch.has(x86_feature::avx512bw) &&
1127 Arch.has(x86_feature::avx512vbmi2) &&
1128 Arch.has(x86_feature::avx512vl))
1129 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1130 __m256i vpshrdvd(
1131 __m256i a,
1132 __m256i b,
1133 __m256i counts) noexcept {
1134 return _mm256_shrdv_epi32(a, b, counts);
1135 }
1136
1137 template<isa<x86> Arch>
1138 requires(Arch.has(x86_feature::avx512f) &&
1139 Arch.has(x86_feature::avx512bw) &&
1140 Arch.has(x86_feature::avx512vbmi2) &&
1141 Arch.has(x86_feature::avx512vl))
1142 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1143 __m256i mask_vpshrdvd(
1144 __m256i a,
1145 __mmask8 mask,
1146 __m256i b,
1147 __m256i counts) noexcept {
1148 return _mm256_mask_shrdv_epi32(a, mask, b, counts);
1149 }
1150
1151 template<isa<x86> Arch>
1152 requires(Arch.has(x86_feature::avx512f) &&
1153 Arch.has(x86_feature::avx512bw) &&
1154 Arch.has(x86_feature::avx512vbmi2) &&
1155 Arch.has(x86_feature::avx512vl))
1156 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1157 __m256i maskz_vpshrdvd(
1158 __mmask8 mask,
1159 __m256i a,
1160 __m256i b,
1161 __m256i counts) noexcept {
1162 return _mm256_maskz_shrdv_epi32(mask, a, b, counts);
1163 }
1164
1165 template<isa<x86> Arch, unsigned Imm8>
1166 requires(Arch.has(x86_feature::avx512f) &&
1167 Arch.has(x86_feature::avx512bw) &&
1168 Arch.has(x86_feature::avx512vbmi2) &&
1169 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1170 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1171 __m256i vpshldq(
1172 __m256i a,
1173 __m256i b) noexcept {
1174 return _mm256_shldi_epi64(a, b, Imm8);
1175 }
1176
1177 template<isa<x86> Arch, unsigned Imm8>
1178 requires(Arch.has(x86_feature::avx512f) &&
1179 Arch.has(x86_feature::avx512bw) &&
1180 Arch.has(x86_feature::avx512vbmi2) &&
1181 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1182 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1183 __m256i mask_vpshldq(
1184 __m256i source,
1185 __mmask8 mask,
1186 __m256i a,
1187 __m256i b) noexcept {
1188 return _mm256_mask_shldi_epi64(source, mask, a, b, Imm8);
1189 }
1190
1191 template<isa<x86> Arch, unsigned Imm8>
1192 requires(Arch.has(x86_feature::avx512f) &&
1193 Arch.has(x86_feature::avx512bw) &&
1194 Arch.has(x86_feature::avx512vbmi2) &&
1195 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1196 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1197 __m256i maskz_vpshldq(
1198 __mmask8 mask,
1199 __m256i a,
1200 __m256i b) noexcept {
1201 return _mm256_maskz_shldi_epi64(mask, a, b, Imm8);
1202 }
1203
1204 template<isa<x86> Arch>
1205 requires(Arch.has(x86_feature::avx512f) &&
1206 Arch.has(x86_feature::avx512bw) &&
1207 Arch.has(x86_feature::avx512vbmi2) &&
1208 Arch.has(x86_feature::avx512vl))
1209 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1210 __m256i vpshldvq(
1211 __m256i a,
1212 __m256i b,
1213 __m256i counts) noexcept {
1214 return _mm256_shldv_epi64(a, b, counts);
1215 }
1216
1217 template<isa<x86> Arch>
1218 requires(Arch.has(x86_feature::avx512f) &&
1219 Arch.has(x86_feature::avx512bw) &&
1220 Arch.has(x86_feature::avx512vbmi2) &&
1221 Arch.has(x86_feature::avx512vl))
1222 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1223 __m256i mask_vpshldvq(
1224 __m256i a,
1225 __mmask8 mask,
1226 __m256i b,
1227 __m256i counts) noexcept {
1228 return _mm256_mask_shldv_epi64(a, mask, b, counts);
1229 }
1230
1231 template<isa<x86> Arch>
1232 requires(Arch.has(x86_feature::avx512f) &&
1233 Arch.has(x86_feature::avx512bw) &&
1234 Arch.has(x86_feature::avx512vbmi2) &&
1235 Arch.has(x86_feature::avx512vl))
1236 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1237 __m256i maskz_vpshldvq(
1238 __mmask8 mask,
1239 __m256i a,
1240 __m256i b,
1241 __m256i counts) noexcept {
1242 return _mm256_maskz_shldv_epi64(mask, a, b, counts);
1243 }
1244
1245 template<isa<x86> Arch, unsigned Imm8>
1246 requires(Arch.has(x86_feature::avx512f) &&
1247 Arch.has(x86_feature::avx512bw) &&
1248 Arch.has(x86_feature::avx512vbmi2) &&
1249 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1250 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1251 __m256i vpshrdq(
1252 __m256i a,
1253 __m256i b) noexcept {
1254 return _mm256_shrdi_epi64(a, b, Imm8);
1255 }
1256
1257 template<isa<x86> Arch, unsigned Imm8>
1258 requires(Arch.has(x86_feature::avx512f) &&
1259 Arch.has(x86_feature::avx512bw) &&
1260 Arch.has(x86_feature::avx512vbmi2) &&
1261 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1262 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1263 __m256i mask_vpshrdq(
1264 __m256i source,
1265 __mmask8 mask,
1266 __m256i a,
1267 __m256i b) noexcept {
1268 return _mm256_mask_shrdi_epi64(source, mask, a, b, Imm8);
1269 }
1270
1271 template<isa<x86> Arch, unsigned Imm8>
1272 requires(Arch.has(x86_feature::avx512f) &&
1273 Arch.has(x86_feature::avx512bw) &&
1274 Arch.has(x86_feature::avx512vbmi2) &&
1275 Arch.has(x86_feature::avx512vl) && Imm8 <= 255)
1276 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1277 __m256i maskz_vpshrdq(
1278 __mmask8 mask,
1279 __m256i a,
1280 __m256i b) noexcept {
1281 return _mm256_maskz_shrdi_epi64(mask, a, b, Imm8);
1282 }
1283
1284 template<isa<x86> Arch>
1285 requires(Arch.has(x86_feature::avx512f) &&
1286 Arch.has(x86_feature::avx512bw) &&
1287 Arch.has(x86_feature::avx512vbmi2) &&
1288 Arch.has(x86_feature::avx512vl))
1289 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1290 __m256i vpshrdvq(
1291 __m256i a,
1292 __m256i b,
1293 __m256i counts) noexcept {
1294 return _mm256_shrdv_epi64(a, b, counts);
1295 }
1296
1297 template<isa<x86> Arch>
1298 requires(Arch.has(x86_feature::avx512f) &&
1299 Arch.has(x86_feature::avx512bw) &&
1300 Arch.has(x86_feature::avx512vbmi2) &&
1301 Arch.has(x86_feature::avx512vl))
1302 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1303 __m256i mask_vpshrdvq(
1304 __m256i a,
1305 __mmask8 mask,
1306 __m256i b,
1307 __m256i counts) noexcept {
1308 return _mm256_mask_shrdv_epi64(a, mask, b, counts);
1309 }
1310
1311 template<isa<x86> Arch>
1312 requires(Arch.has(x86_feature::avx512f) &&
1313 Arch.has(x86_feature::avx512bw) &&
1314 Arch.has(x86_feature::avx512vbmi2) &&
1315 Arch.has(x86_feature::avx512vl))
1316 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2,avx512vl")
1317 __m256i maskz_vpshrdvq(
1318 __mmask8 mask,
1319 __m256i a,
1320 __m256i b,
1321 __m256i counts) noexcept {
1322 return _mm256_maskz_shrdv_epi64(mask, a, b, counts);
1323 }
1324
1325 template<isa<x86> Arch>
1326 requires(Arch.has(x86_feature::avx512f) &&
1327 Arch.has(x86_feature::avx512bw) &&
1328 Arch.has(x86_feature::avx512vbmi2))
1329 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1330 __m512i mask_vpcompressb(
1331 __m512i source,
1332 __mmask64 mask,
1333 __m512i value) noexcept {
1334 return _mm512_mask_compress_epi8(source, mask, value);
1335 }
1336
1337 template<isa<x86> Arch>
1338 requires(Arch.has(x86_feature::avx512f) &&
1339 Arch.has(x86_feature::avx512bw) &&
1340 Arch.has(x86_feature::avx512vbmi2))
1341 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1342 __m512i maskz_vpcompressb(
1343 __mmask64 mask,
1344 __m512i value) noexcept {
1345 return _mm512_maskz_compress_epi8(mask, value);
1346 }
1347
1348 template<isa<x86> Arch>
1349 requires(Arch.has(x86_feature::avx512f) &&
1350 Arch.has(x86_feature::avx512bw) &&
1351 Arch.has(x86_feature::avx512vbmi2))
1352 native_inline native_target("avx512f,avx512bw,avx512vbmi2")
1353 void mask_vpcompressb(
1354 void * destination,
1355 __mmask64 mask,
1356 __m512i value) noexcept {
1357 _mm512_mask_compressstoreu_epi8(destination, mask, value);
1358 }
1359
1360 template<isa<x86> Arch>
1361 requires(Arch.has(x86_feature::avx512f) &&
1362 Arch.has(x86_feature::avx512bw) &&
1363 Arch.has(x86_feature::avx512vbmi2))
1364 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1365 __m512i mask_vpexpandb(
1366 __m512i source,
1367 __mmask64 mask,
1368 __m512i value) noexcept {
1369 return _mm512_mask_expand_epi8(source, mask, value);
1370 }
1371
1372 template<isa<x86> Arch>
1373 requires(Arch.has(x86_feature::avx512f) &&
1374 Arch.has(x86_feature::avx512bw) &&
1375 Arch.has(x86_feature::avx512vbmi2))
1376 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1377 __m512i maskz_vpexpandb(
1378 __mmask64 mask,
1379 __m512i value) noexcept {
1380 return _mm512_maskz_expand_epi8(mask, value);
1381 }
1382
1383 template<isa<x86> Arch>
1384 requires(Arch.has(x86_feature::avx512f) &&
1385 Arch.has(x86_feature::avx512bw) &&
1386 Arch.has(x86_feature::avx512vbmi2))
1387 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2")
1388 __m512i mask_vpexpandb(
1389 __m512i source,
1390 __mmask64 mask,
1391 void const * memory) noexcept {
1392 return _mm512_mask_expandloadu_epi8(source, mask, memory);
1393 }
1394
1395 template<isa<x86> Arch>
1396 requires(Arch.has(x86_feature::avx512f) &&
1397 Arch.has(x86_feature::avx512bw) &&
1398 Arch.has(x86_feature::avx512vbmi2))
1399 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2")
1400 __m512i maskz_vpexpandb_512(
1401 __mmask64 mask,
1402 void const * memory) noexcept {
1403 return _mm512_maskz_expandloadu_epi8(mask, memory);
1404 }
1405
1406 template<isa<x86> Arch>
1407 requires(Arch.has(x86_feature::avx512f) &&
1408 Arch.has(x86_feature::avx512bw) &&
1409 Arch.has(x86_feature::avx512vbmi2))
1410 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1411 __m512i mask_vpcompressw(
1412 __m512i source,
1413 __mmask32 mask,
1414 __m512i value) noexcept {
1415 return _mm512_mask_compress_epi16(source, mask, value);
1416 }
1417
1418 template<isa<x86> Arch>
1419 requires(Arch.has(x86_feature::avx512f) &&
1420 Arch.has(x86_feature::avx512bw) &&
1421 Arch.has(x86_feature::avx512vbmi2))
1422 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1423 __m512i maskz_vpcompressw(
1424 __mmask32 mask,
1425 __m512i value) noexcept {
1426 return _mm512_maskz_compress_epi16(mask, value);
1427 }
1428
1429 template<isa<x86> Arch>
1430 requires(Arch.has(x86_feature::avx512f) &&
1431 Arch.has(x86_feature::avx512bw) &&
1432 Arch.has(x86_feature::avx512vbmi2))
1433 native_inline native_target("avx512f,avx512bw,avx512vbmi2")
1434 void mask_vpcompressw(
1435 void * destination,
1436 __mmask32 mask,
1437 __m512i value) noexcept {
1438 _mm512_mask_compressstoreu_epi16(destination, mask, value);
1439 }
1440
1441 template<isa<x86> Arch>
1442 requires(Arch.has(x86_feature::avx512f) &&
1443 Arch.has(x86_feature::avx512bw) &&
1444 Arch.has(x86_feature::avx512vbmi2))
1445 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1446 __m512i mask_vpexpandw(
1447 __m512i source,
1448 __mmask32 mask,
1449 __m512i value) noexcept {
1450 return _mm512_mask_expand_epi16(source, mask, value);
1451 }
1452
1453 template<isa<x86> Arch>
1454 requires(Arch.has(x86_feature::avx512f) &&
1455 Arch.has(x86_feature::avx512bw) &&
1456 Arch.has(x86_feature::avx512vbmi2))
1457 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1458 __m512i maskz_vpexpandw(
1459 __mmask32 mask,
1460 __m512i value) noexcept {
1461 return _mm512_maskz_expand_epi16(mask, value);
1462 }
1463
1464 template<isa<x86> Arch>
1465 requires(Arch.has(x86_feature::avx512f) &&
1466 Arch.has(x86_feature::avx512bw) &&
1467 Arch.has(x86_feature::avx512vbmi2))
1468 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2")
1469 __m512i mask_vpexpandw(
1470 __m512i source,
1471 __mmask32 mask,
1472 void const * memory) noexcept {
1473 return _mm512_mask_expandloadu_epi16(source, mask, memory);
1474 }
1475
1476 template<isa<x86> Arch>
1477 requires(Arch.has(x86_feature::avx512f) &&
1478 Arch.has(x86_feature::avx512bw) &&
1479 Arch.has(x86_feature::avx512vbmi2))
1480 native_nodiscard native_inline native_pure native_target("avx512f,avx512bw,avx512vbmi2")
1481 __m512i maskz_vpexpandw_512(
1482 __mmask32 mask,
1483 void const * memory) noexcept {
1484 return _mm512_maskz_expandloadu_epi16(mask, memory);
1485 }
1486
1487 template<isa<x86> Arch, unsigned Imm8>
1488 requires(Arch.has(x86_feature::avx512f) &&
1489 Arch.has(x86_feature::avx512bw) &&
1490 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1491 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1492 __m512i vpshldw(
1493 __m512i a,
1494 __m512i b) noexcept {
1495 return _mm512_shldi_epi16(a, b, Imm8);
1496 }
1497
1498 template<isa<x86> Arch, unsigned Imm8>
1499 requires(Arch.has(x86_feature::avx512f) &&
1500 Arch.has(x86_feature::avx512bw) &&
1501 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1502 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1503 __m512i mask_vpshldw(
1504 __m512i source,
1505 __mmask32 mask,
1506 __m512i a,
1507 __m512i b) noexcept {
1508 return _mm512_mask_shldi_epi16(source, mask, a, b, Imm8);
1509 }
1510
1511 template<isa<x86> Arch, unsigned Imm8>
1512 requires(Arch.has(x86_feature::avx512f) &&
1513 Arch.has(x86_feature::avx512bw) &&
1514 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1515 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1516 __m512i maskz_vpshldw(
1517 __mmask32 mask,
1518 __m512i a,
1519 __m512i b) noexcept {
1520 return _mm512_maskz_shldi_epi16(mask, a, b, Imm8);
1521 }
1522
1523 template<isa<x86> Arch>
1524 requires(Arch.has(x86_feature::avx512f) &&
1525 Arch.has(x86_feature::avx512bw) &&
1526 Arch.has(x86_feature::avx512vbmi2))
1527 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1528 __m512i vpshldvw(
1529 __m512i a,
1530 __m512i b,
1531 __m512i counts) noexcept {
1532 return _mm512_shldv_epi16(a, b, counts);
1533 }
1534
1535 template<isa<x86> Arch>
1536 requires(Arch.has(x86_feature::avx512f) &&
1537 Arch.has(x86_feature::avx512bw) &&
1538 Arch.has(x86_feature::avx512vbmi2))
1539 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1540 __m512i mask_vpshldvw(
1541 __m512i a,
1542 __mmask32 mask,
1543 __m512i b,
1544 __m512i counts) noexcept {
1545 return _mm512_mask_shldv_epi16(a, mask, b, counts);
1546 }
1547
1548 template<isa<x86> Arch>
1549 requires(Arch.has(x86_feature::avx512f) &&
1550 Arch.has(x86_feature::avx512bw) &&
1551 Arch.has(x86_feature::avx512vbmi2))
1552 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1553 __m512i maskz_vpshldvw(
1554 __mmask32 mask,
1555 __m512i a,
1556 __m512i b,
1557 __m512i counts) noexcept {
1558 return _mm512_maskz_shldv_epi16(mask, a, b, counts);
1559 }
1560
1561 template<isa<x86> Arch, unsigned Imm8>
1562 requires(Arch.has(x86_feature::avx512f) &&
1563 Arch.has(x86_feature::avx512bw) &&
1564 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1565 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1566 __m512i vpshrdw(
1567 __m512i a,
1568 __m512i b) noexcept {
1569 return _mm512_shrdi_epi16(a, b, Imm8);
1570 }
1571
1572 template<isa<x86> Arch, unsigned Imm8>
1573 requires(Arch.has(x86_feature::avx512f) &&
1574 Arch.has(x86_feature::avx512bw) &&
1575 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1576 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1577 __m512i mask_vpshrdw(
1578 __m512i source,
1579 __mmask32 mask,
1580 __m512i a,
1581 __m512i b) noexcept {
1582 return _mm512_mask_shrdi_epi16(source, mask, a, b, Imm8);
1583 }
1584
1585 template<isa<x86> Arch, unsigned Imm8>
1586 requires(Arch.has(x86_feature::avx512f) &&
1587 Arch.has(x86_feature::avx512bw) &&
1588 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1589 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1590 __m512i maskz_vpshrdw(
1591 __mmask32 mask,
1592 __m512i a,
1593 __m512i b) noexcept {
1594 return _mm512_maskz_shrdi_epi16(mask, a, b, Imm8);
1595 }
1596
1597 template<isa<x86> Arch>
1598 requires(Arch.has(x86_feature::avx512f) &&
1599 Arch.has(x86_feature::avx512bw) &&
1600 Arch.has(x86_feature::avx512vbmi2))
1601 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1602 __m512i vpshrdvw(
1603 __m512i a,
1604 __m512i b,
1605 __m512i counts) noexcept {
1606 return _mm512_shrdv_epi16(a, b, counts);
1607 }
1608
1609 template<isa<x86> Arch>
1610 requires(Arch.has(x86_feature::avx512f) &&
1611 Arch.has(x86_feature::avx512bw) &&
1612 Arch.has(x86_feature::avx512vbmi2))
1613 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1614 __m512i mask_vpshrdvw(
1615 __m512i a,
1616 __mmask32 mask,
1617 __m512i b,
1618 __m512i counts) noexcept {
1619 return _mm512_mask_shrdv_epi16(a, mask, b, counts);
1620 }
1621
1622 template<isa<x86> Arch>
1623 requires(Arch.has(x86_feature::avx512f) &&
1624 Arch.has(x86_feature::avx512bw) &&
1625 Arch.has(x86_feature::avx512vbmi2))
1626 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1627 __m512i maskz_vpshrdvw(
1628 __mmask32 mask,
1629 __m512i a,
1630 __m512i b,
1631 __m512i counts) noexcept {
1632 return _mm512_maskz_shrdv_epi16(mask, a, b, counts);
1633 }
1634
1635 template<isa<x86> Arch, unsigned Imm8>
1636 requires(Arch.has(x86_feature::avx512f) &&
1637 Arch.has(x86_feature::avx512bw) &&
1638 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1639 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1640 __m512i vpshldd(
1641 __m512i a,
1642 __m512i b) noexcept {
1643 return _mm512_shldi_epi32(a, b, Imm8);
1644 }
1645
1646 template<isa<x86> Arch, unsigned Imm8>
1647 requires(Arch.has(x86_feature::avx512f) &&
1648 Arch.has(x86_feature::avx512bw) &&
1649 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1650 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1651 __m512i mask_vpshldd(
1652 __m512i source,
1653 __mmask16 mask,
1654 __m512i a,
1655 __m512i b) noexcept {
1656 return _mm512_mask_shldi_epi32(source, mask, a, b, Imm8);
1657 }
1658
1659 template<isa<x86> Arch, unsigned Imm8>
1660 requires(Arch.has(x86_feature::avx512f) &&
1661 Arch.has(x86_feature::avx512bw) &&
1662 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1663 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1664 __m512i maskz_vpshldd(
1665 __mmask16 mask,
1666 __m512i a,
1667 __m512i b) noexcept {
1668 return _mm512_maskz_shldi_epi32(mask, a, b, Imm8);
1669 }
1670
1671 template<isa<x86> Arch>
1672 requires(Arch.has(x86_feature::avx512f) &&
1673 Arch.has(x86_feature::avx512bw) &&
1674 Arch.has(x86_feature::avx512vbmi2))
1675 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1676 __m512i vpshldvd(
1677 __m512i a,
1678 __m512i b,
1679 __m512i counts) noexcept {
1680 return _mm512_shldv_epi32(a, b, counts);
1681 }
1682
1683 template<isa<x86> Arch>
1684 requires(Arch.has(x86_feature::avx512f) &&
1685 Arch.has(x86_feature::avx512bw) &&
1686 Arch.has(x86_feature::avx512vbmi2))
1687 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1688 __m512i mask_vpshldvd(
1689 __m512i a,
1690 __mmask16 mask,
1691 __m512i b,
1692 __m512i counts) noexcept {
1693 return _mm512_mask_shldv_epi32(a, mask, b, counts);
1694 }
1695
1696 template<isa<x86> Arch>
1697 requires(Arch.has(x86_feature::avx512f) &&
1698 Arch.has(x86_feature::avx512bw) &&
1699 Arch.has(x86_feature::avx512vbmi2))
1700 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1701 __m512i maskz_vpshldvd(
1702 __mmask16 mask,
1703 __m512i a,
1704 __m512i b,
1705 __m512i counts) noexcept {
1706 return _mm512_maskz_shldv_epi32(mask, a, b, counts);
1707 }
1708
1709 template<isa<x86> Arch, unsigned Imm8>
1710 requires(Arch.has(x86_feature::avx512f) &&
1711 Arch.has(x86_feature::avx512bw) &&
1712 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1713 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1714 __m512i vpshrdd(
1715 __m512i a,
1716 __m512i b) noexcept {
1717 return _mm512_shrdi_epi32(a, b, Imm8);
1718 }
1719
1720 template<isa<x86> Arch, unsigned Imm8>
1721 requires(Arch.has(x86_feature::avx512f) &&
1722 Arch.has(x86_feature::avx512bw) &&
1723 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1724 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1725 __m512i mask_vpshrdd(
1726 __m512i source,
1727 __mmask16 mask,
1728 __m512i a,
1729 __m512i b) noexcept {
1730 return _mm512_mask_shrdi_epi32(source, mask, a, b, Imm8);
1731 }
1732
1733 template<isa<x86> Arch, unsigned Imm8>
1734 requires(Arch.has(x86_feature::avx512f) &&
1735 Arch.has(x86_feature::avx512bw) &&
1736 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1737 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1738 __m512i maskz_vpshrdd(
1739 __mmask16 mask,
1740 __m512i a,
1741 __m512i b) noexcept {
1742 return _mm512_maskz_shrdi_epi32(mask, a, b, Imm8);
1743 }
1744
1745 template<isa<x86> Arch>
1746 requires(Arch.has(x86_feature::avx512f) &&
1747 Arch.has(x86_feature::avx512bw) &&
1748 Arch.has(x86_feature::avx512vbmi2))
1749 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1750 __m512i vpshrdvd(
1751 __m512i a,
1752 __m512i b,
1753 __m512i counts) noexcept {
1754 return _mm512_shrdv_epi32(a, b, counts);
1755 }
1756
1757 template<isa<x86> Arch>
1758 requires(Arch.has(x86_feature::avx512f) &&
1759 Arch.has(x86_feature::avx512bw) &&
1760 Arch.has(x86_feature::avx512vbmi2))
1761 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1762 __m512i mask_vpshrdvd(
1763 __m512i a,
1764 __mmask16 mask,
1765 __m512i b,
1766 __m512i counts) noexcept {
1767 return _mm512_mask_shrdv_epi32(a, mask, b, counts);
1768 }
1769
1770 template<isa<x86> Arch>
1771 requires(Arch.has(x86_feature::avx512f) &&
1772 Arch.has(x86_feature::avx512bw) &&
1773 Arch.has(x86_feature::avx512vbmi2))
1774 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1775 __m512i maskz_vpshrdvd(
1776 __mmask16 mask,
1777 __m512i a,
1778 __m512i b,
1779 __m512i counts) noexcept {
1780 return _mm512_maskz_shrdv_epi32(mask, a, b, counts);
1781 }
1782
1783 template<isa<x86> Arch, unsigned Imm8>
1784 requires(Arch.has(x86_feature::avx512f) &&
1785 Arch.has(x86_feature::avx512bw) &&
1786 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1787 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1788 __m512i vpshldq(
1789 __m512i a,
1790 __m512i b) noexcept {
1791 return _mm512_shldi_epi64(a, b, Imm8);
1792 }
1793
1794 template<isa<x86> Arch, unsigned Imm8>
1795 requires(Arch.has(x86_feature::avx512f) &&
1796 Arch.has(x86_feature::avx512bw) &&
1797 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1798 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1799 __m512i mask_vpshldq(
1800 __m512i source,
1801 __mmask8 mask,
1802 __m512i a,
1803 __m512i b) noexcept {
1804 return _mm512_mask_shldi_epi64(source, mask, a, b, Imm8);
1805 }
1806
1807 template<isa<x86> Arch, unsigned Imm8>
1808 requires(Arch.has(x86_feature::avx512f) &&
1809 Arch.has(x86_feature::avx512bw) &&
1810 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1811 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1812 __m512i maskz_vpshldq(
1813 __mmask8 mask,
1814 __m512i a,
1815 __m512i b) noexcept {
1816 return _mm512_maskz_shldi_epi64(mask, a, b, Imm8);
1817 }
1818
1819 template<isa<x86> Arch>
1820 requires(Arch.has(x86_feature::avx512f) &&
1821 Arch.has(x86_feature::avx512bw) &&
1822 Arch.has(x86_feature::avx512vbmi2))
1823 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1824 __m512i vpshldvq(
1825 __m512i a,
1826 __m512i b,
1827 __m512i counts) noexcept {
1828 return _mm512_shldv_epi64(a, b, counts);
1829 }
1830
1831 template<isa<x86> Arch>
1832 requires(Arch.has(x86_feature::avx512f) &&
1833 Arch.has(x86_feature::avx512bw) &&
1834 Arch.has(x86_feature::avx512vbmi2))
1835 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1836 __m512i mask_vpshldvq(
1837 __m512i a,
1838 __mmask8 mask,
1839 __m512i b,
1840 __m512i counts) noexcept {
1841 return _mm512_mask_shldv_epi64(a, mask, b, counts);
1842 }
1843
1844 template<isa<x86> Arch>
1845 requires(Arch.has(x86_feature::avx512f) &&
1846 Arch.has(x86_feature::avx512bw) &&
1847 Arch.has(x86_feature::avx512vbmi2))
1848 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1849 __m512i maskz_vpshldvq(
1850 __mmask8 mask,
1851 __m512i a,
1852 __m512i b,
1853 __m512i counts) noexcept {
1854 return _mm512_maskz_shldv_epi64(mask, a, b, counts);
1855 }
1856
1857 template<isa<x86> Arch, unsigned Imm8>
1858 requires(Arch.has(x86_feature::avx512f) &&
1859 Arch.has(x86_feature::avx512bw) &&
1860 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1861 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1862 __m512i vpshrdq(
1863 __m512i a,
1864 __m512i b) noexcept {
1865 return _mm512_shrdi_epi64(a, b, Imm8);
1866 }
1867
1868 template<isa<x86> Arch, unsigned Imm8>
1869 requires(Arch.has(x86_feature::avx512f) &&
1870 Arch.has(x86_feature::avx512bw) &&
1871 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1872 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1873 __m512i mask_vpshrdq(
1874 __m512i source,
1875 __mmask8 mask,
1876 __m512i a,
1877 __m512i b) noexcept {
1878 return _mm512_mask_shrdi_epi64(source, mask, a, b, Imm8);
1879 }
1880
1881 template<isa<x86> Arch, unsigned Imm8>
1882 requires(Arch.has(x86_feature::avx512f) &&
1883 Arch.has(x86_feature::avx512bw) &&
1884 Arch.has(x86_feature::avx512vbmi2) && Imm8 <= 255)
1885 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1886 __m512i maskz_vpshrdq(
1887 __mmask8 mask,
1888 __m512i a,
1889 __m512i b) noexcept {
1890 return _mm512_maskz_shrdi_epi64(mask, a, b, Imm8);
1891 }
1892
1893 template<isa<x86> Arch>
1894 requires(Arch.has(x86_feature::avx512f) &&
1895 Arch.has(x86_feature::avx512bw) &&
1896 Arch.has(x86_feature::avx512vbmi2))
1897 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1898 __m512i vpshrdvq(
1899 __m512i a,
1900 __m512i b,
1901 __m512i counts) noexcept {
1902 return _mm512_shrdv_epi64(a, b, counts);
1903 }
1904
1905 template<isa<x86> Arch>
1906 requires(Arch.has(x86_feature::avx512f) &&
1907 Arch.has(x86_feature::avx512bw) &&
1908 Arch.has(x86_feature::avx512vbmi2))
1909 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1910 __m512i mask_vpshrdvq(
1911 __m512i a,
1912 __mmask8 mask,
1913 __m512i b,
1914 __m512i counts) noexcept {
1915 return _mm512_mask_shrdv_epi64(a, mask, b, counts);
1916 }
1917
1918 template<isa<x86> Arch>
1919 requires(Arch.has(x86_feature::avx512f) &&
1920 Arch.has(x86_feature::avx512bw) &&
1921 Arch.has(x86_feature::avx512vbmi2))
1922 native_nodiscard native_inline native_const native_target("avx512f,avx512bw,avx512vbmi2")
1923 __m512i maskz_vpshrdvq(
1924 __mmask8 mask,
1925 __m512i a,
1926 __m512i b,
1927 __m512i counts) noexcept {
1928 return _mm512_maskz_shrdv_epi64(mask, a, b, counts);
1929 }
1930}
1931#endif
1932
1933#include <array>
1934#include <concepts>
1935#include <cstddef>
1936#include <cstdint>
1937#include <type_traits>
1938
1939namespace native::detail::x86_vbmi2_constant {
1940 template<bool Expand, class V>
1941 constexpr V compact(V source, std::uint64_t mask, V value) noexcept {
1942 std::array<typename V::value_type, V::lanes> input{}, result{};
1943 value.store(input.data());
1944 source.store(result.data());
1945 std::size_t packed = 0;
1946 for (std::size_t lane = 0; lane < V::lanes; ++lane) {
1947 if ((mask >> lane) & 1) {
1948 if constexpr (Expand) {
1949 result[lane] = input[packed++];
1950 } else {
1951 result[packed++] = input[lane];
1952 }
1953 }
1954 }
1955 return V::load(result.data());
1956 }
1957
1958 template<class T, class V>
1959 constexpr void compress_store(T * destination, std::uint64_t mask, V value) noexcept {
1960 std::array<T, V::lanes> input{};
1961 value.store(input.data());
1962 std::size_t packed = 0;
1963 for (std::size_t lane = 0; lane < V::lanes; ++lane) {
1964 if ((mask >> lane) & 1) {
1965 destination[packed++] = input[lane];
1966 }
1967 }
1968 }
1969
1970 template<class V, class T>
1971 constexpr V expand_load(V source, std::uint64_t mask, T * memory) noexcept {
1972 std::array<typename V::value_type, V::lanes> result{};
1973 source.store(result.data());
1974 std::size_t packed = 0;
1975 for (std::size_t lane = 0; lane < V::lanes; ++lane) {
1976 if ((mask >> lane) & 1) {
1977 result[lane] = memory[packed++];
1978 }
1979 }
1980 return V::load(result.data());
1981 }
1982
1983 template<bool Right, class T>
1984 constexpr T shift_lane(T a, T b, unsigned count) noexcept {
1985 constexpr unsigned bits = sizeof(T) * 8;
1986 count &= bits - 1;
1987 if (!count) {
1988 return a;
1989 }
1990 // Widen narrow unsigned lanes before shifting to avoid signed promotions.
1991 auto first = static_cast<std::uint64_t>(a);
1992 auto second = static_cast<std::uint64_t>(b);
1993 if constexpr (Right) {
1994 return static_cast<T>((first >> count) | (second << (bits - count)));
1995 } else {
1996 return static_cast<T>((first << count) | (second >> (bits - count)));
1997 }
1998 }
1999
2000 template<bool Right, class V, class C>
2001 constexpr V shift(V a, V b, C counts, V source, std::uint64_t mask) noexcept {
2002 using lane_type = typename V::value_type;
2003 std::array<lane_type, V::lanes> first{}, second{}, selectors{}, result{};
2004 a.store(first.data());
2005 b.store(second.data());
2006 source.store(result.data());
2007 if constexpr (!std::is_integral_v<C>) {
2008 counts.store(selectors.data());
2009 }
2010 for (std::size_t lane = 0; lane < V::lanes; ++lane) {
2011 if ((mask >> lane) & 1) {
2012 unsigned count;
2013 if constexpr (std::is_integral_v<C>) {
2014 count = counts;
2015 } else {
2016 count = static_cast<unsigned>(selectors[lane]);
2017 }
2018 result[lane] = shift_lane<Right>(first[lane], second[lane], count);
2019 }
2020 }
2021 return V::load(result.data());
2022 }
2023}
attributes.h
Compiler attributes for host code, with shader-safe shared modifiers.
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native_const
#define native_const
[[const]] is not const
Definition attributes.h:108
native_pure
#define native_pure
[[pure]]
Definition attributes.h:126
native_target
#define native_target(x)
this indicates a required feature set for the current multiversioned function.
Definition attributes.h:476
native::mask
typename mask_traits< std::remove_cvref_t< T > >::type mask
Definition mask_traits.h:22