native 0.0.1
Vectors, masks and wide register packs for C++26
Loading...
Searching...
No Matches
bf16.h
1// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
2#pragma once
4#include "native/config.h"
5#include "native/attributes.h"
6#include "native/isa.h"
7#include "native/arm/detail/register_order.h"
8#if NATIVE_HOST_NEON
9#include <arm_neon.h>
10#endif
11#if NATIVE_HOST_NEON || defined(NATIVE_DOXYGEN)
12namespace native::detail::arm_bf16 {
13
14 // Indexed four-element sources never read the upper register half. Undefined
15 // shuffle lanes keep that unused half from requiring a duplicate/move.
16 // Clang 23 CSEs BF16 intrinsics across FPCR writes, even with FENV_ACCESS,
17 // and discards unused BFMLAL results and their FPSR effects. Volatile assembly
18 // preserves both contracts; the memory clobber orders environment accesses
19 // without adding a CPU memory fence.
20
21 template<isa<arm> Arch>
22 requires(Arch.has(arm_feature::neon_bf16))
23 native_nodiscard native_inline __attribute__((target("bf16")))
24 float32x2_t bfdot(float32x2_t acc, bfloat16x4_t a, bfloat16x4_t b) noexcept {
25 acc = ::native::detail::arm_register_order(acc);
26 a = ::native::detail::arm_register_order(a);
27 auto source = ::native::detail::arm_register_order(b);
28 asm volatile("bfdot %0.2s, %1.4h, %2.4h"
29 : "+w"(acc) : "w"(a), "w"(source) : "memory");
30 return ::native::detail::arm_register_order(acc);
31 }
32
33 template<isa<arm> Arch, unsigned Lane>
34 requires(Arch.has(arm_feature::neon_bf16) && Lane < 2)
35 native_nodiscard native_inline __attribute__((target("bf16")))
36 float32x2_t bfdot_lane(float32x2_t acc, bfloat16x4_t a, bfloat16x4_t b) noexcept {
37 acc = ::native::detail::arm_register_order(acc);
38 a = ::native::detail::arm_register_order(a);
39 auto source = ::native::detail::arm_register_order(__builtin_shufflevector(b, b, 0, 1, 2, 3, -1, -1, -1, -1));
40 asm volatile("bfdot %0.2s, %1.4h, %2.2h[%3]"
41 : "+w"(acc) : "w"(a), "w"(source), "i"(Lane) : "memory");
42 return ::native::detail::arm_register_order(acc);
43 }
44
45 template<isa<arm> Arch, unsigned Lane>
46 requires(Arch.has(arm_feature::neon_bf16) && Lane < 4)
47 native_nodiscard native_inline __attribute__((target("bf16")))
48 float32x2_t bfdot_lane(float32x2_t acc, bfloat16x4_t a, bfloat16x8_t b) noexcept {
49 acc = ::native::detail::arm_register_order(acc);
50 a = ::native::detail::arm_register_order(a);
51 auto source = ::native::detail::arm_register_order(b);
52 asm volatile("bfdot %0.2s, %1.4h, %2.2h[%3]"
53 : "+w"(acc) : "w"(a), "w"(source), "i"(Lane) : "memory");
54 return ::native::detail::arm_register_order(acc);
55 }
56
57 template<isa<arm> Arch>
58 requires(Arch.has(arm_feature::neon_bf16))
59 native_nodiscard native_inline __attribute__((target("bf16")))
60 float32x4_t bfdot(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
61 acc = ::native::detail::arm_register_order(acc);
62 a = ::native::detail::arm_register_order(a);
63 auto source = ::native::detail::arm_register_order(b);
64 asm volatile("bfdot %0.4s, %1.8h, %2.8h"
65 : "+w"(acc) : "w"(a), "w"(source) : "memory");
66 return ::native::detail::arm_register_order(acc);
67 }
68
69 template<isa<arm> Arch, unsigned Lane>
70 requires(Arch.has(arm_feature::neon_bf16) && Lane < 2)
71 native_nodiscard native_inline __attribute__((target("bf16")))
72 float32x4_t bfdot_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x4_t b) noexcept {
73 acc = ::native::detail::arm_register_order(acc);
74 a = ::native::detail::arm_register_order(a);
75 auto source = ::native::detail::arm_register_order(__builtin_shufflevector(b, b, 0, 1, 2, 3, -1, -1, -1, -1));
76 asm volatile("bfdot %0.4s, %1.8h, %2.2h[%3]"
77 : "+w"(acc) : "w"(a), "w"(source), "i"(Lane) : "memory");
78 return ::native::detail::arm_register_order(acc);
79 }
80
81 template<isa<arm> Arch, unsigned Lane>
82 requires(Arch.has(arm_feature::neon_bf16) && Lane < 4)
83 native_nodiscard native_inline __attribute__((target("bf16")))
84 float32x4_t bfdot_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
85 acc = ::native::detail::arm_register_order(acc);
86 a = ::native::detail::arm_register_order(a);
87 auto source = ::native::detail::arm_register_order(b);
88 asm volatile("bfdot %0.4s, %1.8h, %2.2h[%3]"
89 : "+w"(acc) : "w"(a), "w"(source), "i"(Lane) : "memory");
90 return ::native::detail::arm_register_order(acc);
91 }
92
93 template<isa<arm> Arch>
94 requires(Arch.has(arm_feature::neon_bf16))
95 native_nodiscard native_inline __attribute__((target("bf16")))
96 float32x4_t bfmmla(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
97 acc = ::native::detail::arm_register_order(acc);
98 a = ::native::detail::arm_register_order(a);
99 auto source = ::native::detail::arm_register_order(b);
100 asm volatile("bfmmla %0.4s, %1.8h, %2.8h"
101 : "+w"(acc) : "w"(a), "w"(source) : "memory");
102 return ::native::detail::arm_register_order(acc);
103 }
104
105 template<isa<arm> Arch>
106 requires(Arch.has(arm_feature::neon_bf16))
107 native_nodiscard native_inline __attribute__((target("bf16")))
108 float32x4_t bfmlalb(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
109 acc = ::native::detail::arm_register_order(acc);
110 a = ::native::detail::arm_register_order(a);
111 auto source = ::native::detail::arm_register_order(b);
112 asm volatile("bfmlalb %0.4s, %1.8h, %2.8h"
113 : "+w"(acc) : "w"(a), "w"(source) : "memory");
114 return ::native::detail::arm_register_order(acc);
115 }
116
117 template<isa<arm> Arch, unsigned Lane>
118 requires(Arch.has(arm_feature::neon_bf16) && Lane < 4)
119 native_nodiscard native_inline __attribute__((target("bf16")))
120 float32x4_t bfmlalb_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x4_t b) noexcept {
121 acc = ::native::detail::arm_register_order(acc);
122 a = ::native::detail::arm_register_order(a);
123 auto source = ::native::detail::arm_register_order(__builtin_shufflevector(b, b, 0, 1, 2, 3, -1, -1, -1, -1));
124 asm volatile("bfmlalb %0.4s, %1.8h, %2.h[%3]"
125 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
126 return ::native::detail::arm_register_order(acc);
127 }
128
129 template<isa<arm> Arch, unsigned Lane>
130 requires(Arch.has(arm_feature::neon_bf16) && Lane < 8)
131 native_nodiscard native_inline __attribute__((target("bf16")))
132 float32x4_t bfmlalb_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
133 acc = ::native::detail::arm_register_order(acc);
134 a = ::native::detail::arm_register_order(a);
135 auto source = ::native::detail::arm_register_order(b);
136 asm volatile("bfmlalb %0.4s, %1.8h, %2.h[%3]"
137 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
138 return ::native::detail::arm_register_order(acc);
139 }
140
141 template<isa<arm> Arch>
142 requires(Arch.has(arm_feature::neon_bf16))
143 native_nodiscard native_inline __attribute__((target("bf16")))
144 float32x4_t bfmlalt(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
145 acc = ::native::detail::arm_register_order(acc);
146 a = ::native::detail::arm_register_order(a);
147 auto source = ::native::detail::arm_register_order(b);
148 asm volatile("bfmlalt %0.4s, %1.8h, %2.8h"
149 : "+w"(acc) : "w"(a), "w"(source) : "memory");
150 return ::native::detail::arm_register_order(acc);
151 }
152
153 template<isa<arm> Arch, unsigned Lane>
154 requires(Arch.has(arm_feature::neon_bf16) && Lane < 4)
155 native_nodiscard native_inline __attribute__((target("bf16")))
156 float32x4_t bfmlalt_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x4_t b) noexcept {
157 acc = ::native::detail::arm_register_order(acc);
158 a = ::native::detail::arm_register_order(a);
159 auto source = ::native::detail::arm_register_order(__builtin_shufflevector(b, b, 0, 1, 2, 3, -1, -1, -1, -1));
160 asm volatile("bfmlalt %0.4s, %1.8h, %2.h[%3]"
161 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
162 return ::native::detail::arm_register_order(acc);
163 }
164
165 template<isa<arm> Arch, unsigned Lane>
166 requires(Arch.has(arm_feature::neon_bf16) && Lane < 8)
167 native_nodiscard native_inline __attribute__((target("bf16")))
168 float32x4_t bfmlalt_lane(float32x4_t acc, bfloat16x8_t a, bfloat16x8_t b) noexcept {
169 acc = ::native::detail::arm_register_order(acc);
170 a = ::native::detail::arm_register_order(a);
171 auto source = ::native::detail::arm_register_order(b);
172 asm volatile("bfmlalt %0.4s, %1.8h, %2.h[%3]"
173 : "+w"(acc) : "w"(a), "x"(source), "i"(Lane) : "memory");
174 return ::native::detail::arm_register_order(acc);
175 }
176
177 // Exact deduction rejects unrelated vectors and invalid immediates before
178 // Clang's lax vector conversions can select an overload for another shape.
179 template<isa<arm> Arch, class R, class A, class B> void bfdot(R, A, B) = delete;
180 template<isa<arm> Arch, class R, class A, class B> void bfmmla(R, A, B) = delete;
181 template<isa<arm> Arch, class R, class A, class B> void bfmlalb(R, A, B) = delete;
182 template<isa<arm> Arch, class R, class A, class B> void bfmlalt(R, A, B) = delete;
183 template<isa<arm> Arch, unsigned Lane, class R, class A, class B> void bfdot_lane(R, A, B) = delete;
184 template<isa<arm> Arch, unsigned Lane, class R, class A, class B> void bfmlalb_lane(R, A, B) = delete;
185 template<isa<arm> Arch, unsigned Lane, class R, class A, class B> void bfmlalt_lane(R, A, B) = delete;
186
187}
188#endif
attributes.h
Compiler attributes for host code, with shader-safe shared modifiers.
native::bfmlalt_lane
constexpr simd< float, 4, Arch > bfmlalt_lane(simd< float, 4, Arch > acc, simd< bf16, 8, Arch > a, simd< bf16, 4, Arch > b) noexcept
Fused multiply-add of the odd lanes of a with b[Lane] shared by all output lanes.
Definition native.arm.bf16.ccm:284
native::bfdot_lane
constexpr simd< float, 2, Arch > bfdot_lane(simd< float, 2, Arch > acc, simd< bf16, 4, Arch > a, simd< bf16, 4, Arch > b) noexcept
BFDOT with the BF16 pair b[2*Lane], b[2*Lane+1] shared by all output lanes.
Definition native.arm.bf16.ccm:92
native::bfmlalb
constexpr simd< float, 4, Arch > bfmlalb(simd< float, 4, Arch > acc, simd< bf16, 8, Arch > a, simd< bf16, 8, Arch > b) noexcept
Fused multiply-add of the even BF16 lanes into the corresponding FP32 lanes.
Definition native.arm.bf16.ccm:208
native::bfmlalb_lane
constexpr simd< float, 4, Arch > bfmlalb_lane(simd< float, 4, Arch > acc, simd< bf16, 8, Arch > a, simd< bf16, 4, Arch > b) noexcept
Fused multiply-add of the even lanes of a with b[Lane] shared by all output lanes.
Definition native.arm.bf16.ccm:227
native::bfmmla
constexpr simd< float, 4, Arch > bfmmla(simd< float, 4, Arch > acc, simd< bf16, 8, Arch > a, simd< bf16, 8, Arch > b) noexcept
Accumulate a row-major 2x4 matrix times a column-major 4x2 matrix, two BFDOT steps per result.
Definition native.arm.bf16.ccm:189
native::bfmlalt
constexpr simd< float, 4, Arch > bfmlalt(simd< float, 4, Arch > acc, simd< bf16, 8, Arch > a, simd< bf16, 8, Arch > b) noexcept
Fused multiply-add of the odd BF16 lanes into the corresponding FP32 lanes.
Definition native.arm.bf16.ccm:265
native::bfdot
constexpr simd< float, 2, Arch > bfdot(simd< float, 2, Arch > acc, simd< bf16, 4, Arch > a, simd< bf16, 4, Arch > b) noexcept
Accumulate each adjacent pair of BF16 products into the corresponding FP32 lane.
Definition native.arm.bf16.ccm:72
native_inline
#define native_inline
inline [[always_inline]]
Definition attributes.h:212
native_nodiscard
#define native_nodiscard
C++17 [[nodiscard]].
Definition attributes.h:189
native::target
constexpr int target
First matching requirement, with every later choice checked for shadowing.
Definition isa.h:396