NEON: implement all bf16-related intrinsics (#1110)

* [Feat] Add BF16 when the machine is supported.

Finished: vld1_bf16_x4 and vld1q_bf16_x2

* [NEON] Add a C implementation of the bf16 type

* [NEON] Add all ld_*_bf16 intrinsics.

* [NEON] Add all st*_bf16 intrinsics.

* [Test] Add vbfdot_f32 test case

* [NEON] Complete converting function from float32 to bfloat16.

- Also add bf-related functions in three series
- cvt, dot, dot_lane

* [Feat] Add option '+bf16' in cross-file

* [NEON] Completed initial implementation of bf-16 related intrinsics.

* [Fix] Remove redundant commment

* [Fix] Correct native aliases

* [Fix] The test generation code has been completed.

arm/neon/combine.h

@@ -442,6 +442,31 @@ simde_vcombine_p64(simde_poly64x1_t low, simde_poly64x1_t high) {
   #define vcombine_p64(low, high) simde_vcombine_p64((low), (high))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vcombine_bf16(simde_bfloat16x4_t low, simde_bfloat16x4_t high) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcombine_bf16(low, high);
+  #else
+    simde_bfloat16x8_private r_;
+    simde_bfloat16x4_private
+      low_ = simde_bfloat16x4_to_private(low),
+      high_ = simde_bfloat16x4_to_private(high);
+
+    size_t halfway = (sizeof(r_.values) / sizeof(r_.values[0])) / 2;
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < halfway ; i++) {
+      r_.values[i] = low_.values[i];
+      r_.values[i + halfway] = high_.values[i];
+    }
+
+    return simde_bfloat16x8_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcombine_bf16
+  #define vcombine_bf16(low, high) simde_vcombine_bf16((low), (high))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/copy_lane.h

@@ -1100,6 +1100,83 @@ simde_vcopyq_laneq_p64(simde_poly64x2_t a, const int lane1, simde_poly64x2_t b,
   #define vcopyq_laneq_p64(a, lane1, b, lane2) simde_vcopyq_laneq_p64((a), (lane1), (b), (lane2))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vcopy_lane_bf16(simde_bfloat16x4_t a, const int lane1, simde_bfloat16x4_t b, const int lane2)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane1, 0, 3)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane2, 0, 3) {
+  simde_bfloat16x4_private
+    b_ = simde_bfloat16x4_to_private(b),
+    r_ = simde_bfloat16x4_to_private(a);
+
+  r_.values[lane1] = b_.values[lane2];
+  return simde_bfloat16x4_from_private(r_);
+}
+#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vcopy_lane_bf16(a, lane1, b, lane2) vcopy_lane_bf16((a), (lane1), (b), (lane2))
+#endif
+#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
+  #undef vcopy_lane_bf16
+  #define vcopy_lane_bf16(a, lane1, b, lane2) simde_vcopy_lane_bf16((a), (lane1), (b), (lane2))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vcopy_laneq_bf16(simde_bfloat16x4_t a, const int lane1, simde_bfloat16x8_t b, const int lane2)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane1, 0, 3)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane2, 0, 7) {
+  simde_bfloat16x4_private r_ = simde_bfloat16x4_to_private(a);
+  simde_bfloat16x8_private b_ = simde_bfloat16x8_to_private(b);
+
+  r_.values[lane1] = b_.values[lane2];
+  return simde_bfloat16x4_from_private(r_);
+}
+#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vcopy_laneq_bf16(a, lane1, b, lane2) vcopy_laneq_bf16((a), (lane1), (b), (lane2))
+#endif
+#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
+  #undef vcopy_laneq_bf16
+  #define vcopy_laneq_bf16(a, lane1, b, lane2) simde_vcopy_laneq_bf16((a), (lane1), (b), (lane2))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vcopyq_lane_bf16(simde_bfloat16x8_t a, const int lane1, simde_bfloat16x4_t b, const int lane2)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane1, 0, 7)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane2, 0, 3) {
+  simde_bfloat16x4_private b_ = simde_bfloat16x4_to_private(b);
+  simde_bfloat16x8_private r_ = simde_bfloat16x8_to_private(a);
+
+  r_.values[lane1] = b_.values[lane2];
+  return simde_bfloat16x8_from_private(r_);
+}
+#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vcopyq_lane_bf16(a, lane1, b, lane2) vcopyq_lane_bf16((a), (lane1), (b), (lane2))
+#endif
+#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
+  #undef vcopyq_lane_bf16
+  #define vcopyq_lane_bf16(a, lane1, b, lane2) simde_vcopyq_lane_bf16((a), (lane1), (b), (lane2))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vcopyq_laneq_bf16(simde_bfloat16x8_t a, const int lane1, simde_bfloat16x8_t b, const int lane2)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane1, 0, 7)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane2, 0, 7) {
+  simde_bfloat16x8_private
+    b_ = simde_bfloat16x8_to_private(b),
+    r_ = simde_bfloat16x8_to_private(a);
+
+  r_.values[lane1] = b_.values[lane2];
+  return simde_bfloat16x8_from_private(r_);
+}
+#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vcopyq_laneq_bf16(a, lane1, b, lane2) vcopyq_laneq_bf16((a), (lane1), (b), (lane2))
+#endif
+#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
+  #undef vcopyq_laneq_bf16
+  #define vcopyq_laneq_bf16(a, lane1, b, lane2) simde_vcopyq_laneq_bf16((a), (lane1), (b), (lane2))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/create.h

@@ -26,8 +26,6 @@
  *   2023      Yi-Yen Chung <eric681@andestech.com> (Copyright owned by Andes Technology)
  */
 
-/* Yi-Yen Chung: Added vcreate_f16 */
-
 #if !defined(SIMDE_ARM_NEON_CREATE_H)
 #define SIMDE_ARM_NEON_CREATE_H
 
@@ -235,6 +233,19 @@ simde_vcreate_p64(simde_poly64_t a) {
   #define vcreate_p64(a) simde_vcreate_p64(a)
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vcreate_bf16(uint64_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcreate_bf16(a);
+  #else
+    return simde_vreinterpret_bf16_u64(simde_vdup_n_u64(a));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcreate_bf16
+  #define vcreate_bf16(a) simde_vcreate_bf16(a)
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/cvt.h

@@ -2068,6 +2068,170 @@ simde_vcvtx_high_f32_f64(simde_float32x2_t r, simde_float64x2_t a) {
   #define vcvtx_high_f32_f64(r, a) simde_vcvtx_high_f32_f64((r), (a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vcvt_bf16_f32(simde_float32x4_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvt_bf16_f32(a);
+  #else
+    simde_float32x4_private a_ = simde_float32x4_to_private(a);
+    simde_bfloat16x4_private r_;
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = simde_bfloat16_from_float32(a_.values[i]);
+    }
+
+    return simde_bfloat16x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvt_bf16_f32
+  #define vcvt_bf16_f32(a) simde_vcvt_bf16_f32(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vcvt_f32_bf16(simde_bfloat16x4_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvt_f32_bf16(a);
+  #else
+    simde_bfloat16x4_private a_ = simde_bfloat16x4_to_private(a);
+    simde_float32x4_private r_;
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = simde_bfloat16_to_float32(a_.values[i]);
+    }
+
+    return simde_float32x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvt_f32_bf16
+  #define vcvt_f32_bf16(a) simde_vcvt_f32_bf16(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32_t
+simde_vcvtah_f32_bf16(simde_bfloat16_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvtah_f32_bf16(a);
+  #else
+    return simde_bfloat16_to_float32(a);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvtah_f32_bf16
+  #define vcvtah_f32_bf16(a) simde_vcvtah_f32_bf16(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16_t
+simde_vcvth_bf16_f32(float a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvth_bf16_f32(a);
+  #else
+    return simde_bfloat16_from_float32(a);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvth_bf16_f32
+  #define vcvth_bf16_f32(a) simde_vcvth_bf16_f32(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vcvtq_low_f32_bf16(simde_bfloat16x8_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvtq_low_f32_bf16(a);
+  #else
+    simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+    simde_float32x4_private r_;
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = simde_bfloat16_to_float32(a_.values[i]);
+    }
+
+    return simde_float32x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvtq_low_f32_bf16
+  #define vcvtq_low_f32_bf16(a) simde_vcvtq_low_f32_bf16(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vcvtq_high_f32_bf16(simde_bfloat16x8_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvtq_high_f32_bf16(a);
+  #else
+    simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+    simde_float32x4_private r_;
+
+    size_t rsize = (sizeof(r_.values) / sizeof(r_.values[0]));
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = simde_bfloat16_to_float32(a_.values[i + rsize]);
+    }
+
+    return simde_float32x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvtq_high_f32_bf16
+  #define vcvtq_high_f32_bf16(a) simde_vcvtq_high_f32_bf16(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vcvtq_low_bf16_f32(simde_float32x4_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvtq_low_bf16_f32(a);
+  #else
+    simde_float32x4_private a_ = simde_float32x4_to_private(a);
+    simde_bfloat16x8_private r_;
+
+    size_t asize = (sizeof(a_.values) / sizeof(a_.values[0]));
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < asize; i++) {
+      r_.values[i] = simde_bfloat16_from_float32(a_.values[i]);
+      r_.values[i + asize] = SIMDE_BFLOAT16_VALUE(0.0);
+    }
+
+    return simde_bfloat16x8_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvtq_low_bf16_f32
+  #define vcvtq_low_bf16_f32(a) simde_vcvtq_low_bf16_f32(a)
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vcvtq_high_bf16_f32(simde_bfloat16x8_t inactive, simde_float32x4_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vcvtq_high_bf16_f32(inactive, a);
+  #else
+    simde_bfloat16x8_private inactive_ = simde_bfloat16x8_to_private(inactive);
+    simde_float32x4_private a_ = simde_float32x4_to_private(a);
+    simde_bfloat16x8_private r_;
+
+    size_t asize = (sizeof(a_.values) / sizeof(a_.values[0]));
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(a_.values) / sizeof(a_.values[0])) ; i++) {
+      r_.values[i] = inactive_.values[i];
+      r_.values[i + asize] = simde_bfloat16_from_float32(a_.values[i]);
+    }
+    return simde_bfloat16x8_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vcvtq_high_bf16_f32
+  #define vcvtq_high_bf16_f32(inactive, a) simde_vcvtq_high_bf16_f32((inactive), (a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/dot.h

@@ -67,7 +67,7 @@ simde_vdot_s32(simde_int32x2_t r, simde_int8x8_t a, simde_int8x8_t b) {
     return simde_vadd_s32(r, simde_int32x2_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_s32
   #define vdot_s32(r, a, b) simde_vdot_s32((r), (a), (b))
 #endif
@@ -97,7 +97,7 @@ simde_vdot_u32(simde_uint32x2_t r, simde_uint8x8_t a, simde_uint8x8_t b) {
     return simde_vadd_u32(r, simde_uint32x2_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_u32
   #define vdot_u32(r, a, b) simde_vdot_u32((r), (a), (b))
 #endif
@@ -128,7 +128,7 @@ simde_vdotq_s32(simde_int32x4_t r, simde_int8x16_t a, simde_int8x16_t b) {
     return simde_vaddq_s32(r, simde_int32x4_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_s32
   #define vdotq_s32(r, a, b) simde_vdotq_s32((r), (a), (b))
 #endif
@@ -159,11 +159,64 @@ simde_vdotq_u32(simde_uint32x4_t r, simde_uint8x16_t a, simde_uint8x16_t b) {
     return simde_vaddq_u32(r, simde_uint32x4_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_u32
   #define vdotq_u32(r, a, b) simde_vdotq_u32((r), (a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x2_t
+simde_vbfdot_f32(simde_float32x2_t r, simde_bfloat16x4_t a, simde_bfloat16x4_t b) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    return vbfdot_f32(r, a, b);
+  #else
+    simde_float32x2_private r_ = simde_float32x2_to_private(r);
+    simde_bfloat16x4_private
+      a_ = simde_bfloat16x4_to_private(a),
+      b_ = simde_bfloat16x4_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * i + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * i + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+    return simde_float32x2_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdot_f32
+  #define vbfdot_f32(r, a, b) simde_vbfdot_f32((r), (a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfdotq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_DOTPROD) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    return vbfdotq_f32(r, a, b);
+  #else
+    simde_float32x4_private r_ = simde_float32x4_to_private(r);
+    simde_bfloat16x8_private
+      a_ = simde_bfloat16x8_to_private(a),
+      b_ = simde_bfloat16x8_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * i + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * i + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+    return simde_float32x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdotq_f32
+  #define vbfdotq_f32(r, a, b) simde_vbfdotq_f32((r), (a), (b))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/dot_lane.h

@@ -86,7 +86,7 @@ simde_vdot_lane_s32(simde_int32x2_t r, simde_int8x8_t a, simde_int8x8_t b, const
 
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_lane_s32
   #define vdot_lane_s32(r, a, b, lane) simde_vdot_lane_s32((r), (a), (b), (lane))
 #endif
@@ -137,7 +137,7 @@ simde_vdot_lane_u32(simde_uint32x2_t r, simde_uint8x8_t a, simde_uint8x8_t b, co
 
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_lane_u32
   #define vdot_lane_u32(r, a, b, lane) simde_vdot_lane_u32((r), (a), (b), (lane))
 #endif
@@ -186,7 +186,7 @@ simde_vdot_laneq_s32(simde_int32x2_t r, simde_int8x8_t a, simde_int8x16_t b, con
 
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_laneq_s32
   #define vdot_laneq_s32(r, a, b, lane) simde_vdot_laneq_s32((r), (a), (b), (lane))
 #endif
@@ -234,7 +234,7 @@ simde_vdot_laneq_u32(simde_uint32x2_t r, simde_uint8x8_t a, simde_uint8x16_t b,
   #endif
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdot_laneq_u32
   #define vdot_laneq_u32(r, a, b, lane) simde_vdot_laneq_u32((r), (a), (b), (lane))
 #endif
@@ -296,7 +296,7 @@ simde_vdotq_laneq_u32(simde_uint32x4_t r, simde_uint8x16_t a, simde_uint8x16_t b
   #endif
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_laneq_u32
   #define vdotq_laneq_u32(r, a, b, lane) simde_vdotq_laneq_u32((r), (a), (b), (lane))
 #endif
@@ -358,7 +358,7 @@ simde_vdotq_laneq_s32(simde_int32x4_t r, simde_int8x16_t a, simde_int8x16_t b, c
   #endif
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_laneq_s32
   #define vdotq_laneq_s32(r, a, b, lane) simde_vdotq_laneq_s32((r), (a), (b), (lane))
 #endif
@@ -419,7 +419,7 @@ simde_vdotq_lane_u32(simde_uint32x4_t r, simde_uint8x16_t a, simde_uint8x8_t b,
   #endif
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_lane_u32
   #define vdotq_lane_u32(r, a, b, lane) simde_vdotq_lane_u32((r), (a), (b), (lane))
 #endif
@@ -480,11 +480,136 @@ simde_vdotq_lane_s32(simde_int32x4_t r, simde_int8x16_t a, simde_int8x8_t b, con
   #endif
   return result;
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_DOTPROD))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vdotq_lane_s32
   #define vdotq_lane_s32(r, a, b, lane) simde_vdotq_lane_s32((r), (a), (b), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x2_t
+simde_vbfdot_lane_f32(simde_float32x2_t r, simde_bfloat16x4_t a, simde_bfloat16x4_t b, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 1) {
+  simde_float32x2_t result;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_DOTPROD) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_2_(vbfdot_lane_f32, result, (HEDLEY_UNREACHABLE(), result), lane, r, a, b);
+  #else
+    simde_float32x2_private r_ = simde_float32x2_to_private(r);
+    simde_bfloat16x4_private
+      a_ = simde_bfloat16x4_to_private(a),
+      b_ = simde_bfloat16x4_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * lane + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * lane + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+
+    result = simde_float32x2_from_private(r_);
+  #endif
+
+  return result;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdot_lane_f32
+  #define vbfdot_lane_f32(r, a, b, lane) simde_vbfdot_lane_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfdotq_lane_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x4_t b, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 1) {
+  simde_float32x4_t result;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_DOTPROD) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_2_(vbfdotq_lane_f32, result, (HEDLEY_UNREACHABLE(), result), lane, r, a, b);
+  #else
+    simde_float32x4_private r_ = simde_float32x4_to_private(r);
+    simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+    simde_bfloat16x4_private b_ = simde_bfloat16x4_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * lane + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * lane + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+
+    result = simde_float32x4_from_private(r_);
+  #endif
+
+  return result;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdotq_lane_f32
+  #define vbfdotq_lane_f32(r, a, b, lane) simde_vbfdotq_lane_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x2_t
+simde_vbfdot_laneq_f32(simde_float32x2_t r, simde_bfloat16x4_t a, simde_bfloat16x8_t b, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_float32x2_t result;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_DOTPROD) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_(vbfdot_laneq_f32, result, (HEDLEY_UNREACHABLE(), result), lane, r, a, b);
+  #else
+    simde_float32x2_private r_ = simde_float32x2_to_private(r);
+    simde_bfloat16x4_private a_ = simde_bfloat16x4_to_private(a);
+    simde_bfloat16x8_private b_ = simde_bfloat16x8_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * lane + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * lane + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+
+    result = simde_float32x2_from_private(r_);
+  #endif
+
+  return result;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdot_laneq_f32
+  #define vbfdot_laneq_f32(r, a, b, lane) simde_vbfdot_laneq_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfdotq_laneq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_float32x4_t result;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_DOTPROD) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_(vbfdotq_laneq_f32, result, (HEDLEY_UNREACHABLE(), result), lane, r, a, b);
+  #else
+    simde_float32x4_private r_ = simde_float32x4_to_private(r);
+    simde_bfloat16x8_private
+      a_ = simde_bfloat16x8_to_private(a),
+      b_ = simde_bfloat16x8_to_private(b);
+
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      simde_float32_t elt1_a = simde_bfloat16_to_float32(a_.values[2 * i + 0]);
+      simde_float32_t elt1_b = simde_bfloat16_to_float32(a_.values[2 * i + 1]);
+      simde_float32_t elt2_a = simde_bfloat16_to_float32(b_.values[2 * lane + 0]);
+      simde_float32_t elt2_b = simde_bfloat16_to_float32(b_.values[2 * lane + 1]);
+      r_.values[i] = r_.values[i] + elt1_a * elt2_a + elt1_b * elt2_b;
+    }
+
+    result = simde_float32x4_from_private(r_);
+  #endif
+
+  return result;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfdotq_laneq_f32
+  #define vbfdotq_laneq_f32(r, a, b, lane) simde_vbfdotq_laneq_f32((r), (a), (b), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/dup_lane.h

@@ -1612,6 +1612,86 @@ simde_vduph_laneq_p16(simde_poly16x8_t vec, const int lane)
   #define vduph_laneq_p16(vec, lane) simde_vduph_laneq_p16((vec), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16_t
+simde_vduph_lane_bf16(simde_bfloat16x4_t vec, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  return simde_bfloat16x4_to_private(vec).values[lane];
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vduph_lane_bf16(vec, lane) vduph_lane_bf16(vec, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vduph_lane_bf16
+  #define vduph_lane_bf16(vec, lane) simde_vduph_lane_bf16((vec), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16_t
+simde_vduph_laneq_bf16(simde_bfloat16x8_t vec, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  return simde_bfloat16x8_to_private(vec).values[lane];
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vduph_laneq_bf16(vec, lane) vduph_laneq_bf16(vec, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vduph_laneq_bf16
+  #define vduph_laneq_bf16(vec, lane) simde_vduph_laneq_bf16((vec), (lane))
+#endif
+
+// simde_vdup_lane_bf16
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vdup_lane_bf16(vec, lane) vdup_lane_bf16(vec, lane)
+#else
+  #define simde_vdup_lane_bf16(vec, lane) simde_vdup_n_bf16(simde_vduph_lane_bf16(vec, lane))
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdup_lane_bf16
+  #define vdup_lane_bf16(vec, lane) simde_vdup_lane_bf16((vec), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vdup_laneq_bf16(simde_bfloat16x8_t vec, const int lane)
+  SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  return simde_vdup_n_bf16(simde_bfloat16x8_to_private(vec).values[lane]);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+#define simde_vdup_laneq_bf16(vec, lane) vdup_laneq_bf16(vec, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdup_laneq_bf16
+  #define vdup_laneq_bf16(vec, lane) simde_vdup_laneq_bf16((vec), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vdupq_lane_bf16(simde_bfloat16x4_t vec, const int lane)
+  SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  return simde_vdupq_n_bf16(simde_bfloat16x4_to_private(vec).values[lane]);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+#define simde_vdupq_lane_bf16(vec, lane) vdupq_lane_bf16(vec, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdupq_lane_bf16
+  #define vdupq_lane_bf16(vec, lane) simde_vdupq_lane_bf16((vec), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vdupq_laneq_bf16(simde_bfloat16x8_t vec, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  return simde_vdupq_n_bf16(simde_bfloat16x8_to_private(vec).values[lane]);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vdupq_laneq_bf16(vec, lane) vdupq_laneq_bf16(vec, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdupq_laneq_bf16
+  #define vdupq_laneq_bf16(vec, lane) simde_vdupq_laneq_bf16((vec), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/dup_n.h

@@ -807,6 +807,47 @@ simde_vdupq_n_p64(simde_poly64_t value) {
   #define vdupq_n_p64(value) simde_vdupq_n_p64((value))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vdup_n_bf16(simde_bfloat16_t value) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vdup_n_bf16(value);
+  #else
+    simde_bfloat16x4_private r_;
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = value;
+    }
+
+    return simde_bfloat16x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdup_n_bf16
+  #define vdup_n_bf16(value) simde_vdup_n_bf16((value))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vdupq_n_bf16(simde_bfloat16_t value) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vdupq_n_bf16(value);
+  #else
+    simde_bfloat16x8_private r_;
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = value;
+    }
+
+    return simde_bfloat16x8_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vdupq_n_bf16
+  #define vdupq_n_bf16(value) simde_vdupq_n_bf16((value))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/fmlal.h

@@ -367,6 +367,159 @@ simde_vfmlalq_laneq_high_f16(simde_float32x4_t r, simde_float16x8_t a, simde_flo
   #define vfmlalq_laneq_high_f16(r, a, b, lane) simde_vfmlalq_laneq_high_f16((r), (a), (b), (lane));
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlalbq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vbfmlalbq_f32(r, a, b);
+  #else
+    simde_float32x4_private
+      ret,
+      r_ = simde_float32x4_to_private(r);
+    simde_bfloat16x8_private
+      a_ = simde_bfloat16x8_to_private(a),
+      b_ = simde_bfloat16x8_to_private(b);
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+      ret.values[i] = r_.values[i] +
+        simde_bfloat16_to_float32(a_.values[i * 2]) * simde_bfloat16_to_float32(b_.values[i * 2]);
+    }
+    return simde_float32x4_from_private(ret);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlalbq_f32
+  #define vbfmlalbq_f32(r, a, b) simde_vbfmlalbq_f32((r), (a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlaltq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vbfmlaltq_f32(r, a, b);
+  #else
+    simde_float32x4_private
+      ret,
+      r_ = simde_float32x4_to_private(r);
+    simde_bfloat16x8_private
+      a_ = simde_bfloat16x8_to_private(a),
+      b_ = simde_bfloat16x8_to_private(b);
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+      ret.values[i] = r_.values[i] +
+        simde_bfloat16_to_float32(a_.values[i * 2 + 1]) * simde_bfloat16_to_float32(b_.values[i * 2 + 1]);
+    }
+    return simde_float32x4_from_private(ret);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlaltq_f32
+  #define vbfmlaltq_f32(r, a, b) simde_vbfmlaltq_f32((r), (a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlalbq_lane_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x4_t b, const int lane)
+   SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_float32x4_private
+    ret,
+    r_ = simde_float32x4_to_private(r);
+  simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+  simde_bfloat16x4_private b_ = simde_bfloat16x4_to_private(b);
+
+  SIMDE_VECTORIZE
+  for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+    ret.values[i] = r_.values[i] +
+      simde_bfloat16_to_float32(a_.values[i * 2]) * simde_bfloat16_to_float32(b_.values[lane]);
+  }
+  return simde_float32x4_from_private(ret);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vbfmlalbq_lane_f32(r, a, b, lane) vbfmlalbq_lane_f32((r), (a), (b), (lane))
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlalbq_lane_f32
+  #define vbfmlalbq_lane_f32(r, a, b, lane) simde_vbfmlalbq_lane_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlalbq_laneq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b, const int lane)
+   SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_float32x4_private
+    ret,
+    r_ = simde_float32x4_to_private(r);
+  simde_bfloat16x8_private
+    a_ = simde_bfloat16x8_to_private(a),
+    b_ = simde_bfloat16x8_to_private(b);
+
+  SIMDE_VECTORIZE
+  for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+    ret.values[i] = r_.values[i] +
+      simde_bfloat16_to_float32(a_.values[i * 2]) * simde_bfloat16_to_float32(b_.values[lane]);
+  }
+  return simde_float32x4_from_private(ret);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vbfmlalbq_laneq_f32(r, a, b, lane) vbfmlalbq_laneq_f32((r), (a), (b), (lane))
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlalbq_laneq_f32
+  #define vbfmlalbq_laneq_f32(r, a, b, lane) simde_vbfmlalbq_laneq_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlaltq_lane_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x4_t b, const int lane)
+   SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_float32x4_private
+    ret,
+    r_ = simde_float32x4_to_private(r);
+  simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+  simde_bfloat16x4_private b_ = simde_bfloat16x4_to_private(b);
+
+  SIMDE_VECTORIZE
+  for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+    ret.values[i] = r_.values[i] +
+      simde_bfloat16_to_float32(a_.values[i * 2 + 1]) * simde_bfloat16_to_float32(b_.values[lane]);
+  }
+  return simde_float32x4_from_private(ret);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vbfmlaltq_lane_f32(r, a, b, lane) vbfmlaltq_lane_f32((r), (a), (b), (lane))
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlaltq_lane_f32
+  #define vbfmlaltq_lane_f32(r, a, b, lane) simde_vbfmlaltq_lane_f32((r), (a), (b), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmlaltq_laneq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b, const int lane)
+   SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_float32x4_private
+    ret,
+    r_ = simde_float32x4_to_private(r);
+  simde_bfloat16x8_private
+    a_ = simde_bfloat16x8_to_private(a),
+    b_ = simde_bfloat16x8_to_private(b);
+
+  SIMDE_VECTORIZE
+  for (size_t i = 0 ; i < (sizeof(ret.values) / sizeof(ret.values[0])) ; i++) {
+    ret.values[i] = r_.values[i] +
+      simde_bfloat16_to_float32(a_.values[i * 2 + 1]) * simde_bfloat16_to_float32(b_.values[lane]);
+  }
+  return simde_float32x4_from_private(ret);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vbfmlaltq_laneq_f32(r, a, b, lane) vbfmlaltq_laneq_f32((r), (a), (b), (lane))
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmlaltq_laneq_f32
+  #define vbfmlaltq_laneq_f32(r, a, b, lane) simde_vbfmlaltq_laneq_f32((r), (a), (b), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/get_high.h

@@ -383,6 +383,27 @@ simde_vget_high_p64(simde_poly64x2_t a) {
   #define vget_high_p64(a) simde_vget_high_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vget_high_bf16(simde_bfloat16x8_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vget_high_bf16(a);
+  #else
+    simde_bfloat16x4_private r_;
+    simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = a_.values[i + (sizeof(r_.values) / sizeof(r_.values[0]))];
+    }
+
+    return simde_bfloat16x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vget_high_bf16
+  #define vget_high_bf16(a) simde_vget_high_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/get_lane.h

@@ -669,6 +669,47 @@ simde_vgetq_lane_p64(simde_poly64x2_t v, const int lane)
   #define vgetq_lane_p64(v, lane) simde_vgetq_lane_p64((v), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16_t
+simde_vget_lane_bf16(simde_bfloat16x4_t v, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16_t r;
+
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_(vget_lane_bf16, r, (HEDLEY_UNREACHABLE(), SIMDE_BFLOAT16_VALUE(0.0)), lane, v);
+  #else
+    simde_bfloat16x4_private v_ = simde_bfloat16x4_to_private(v);
+
+    r = v_.values[lane];
+  #endif
+
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vget_lane_bf16
+  #define vget_lane_bf16(v, lane) simde_vget_lane_bf16((v), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16_t
+simde_vgetq_lane_bf16(simde_bfloat16x8_t v, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16_t r;
+
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_(vgetq_lane_bf16, r, (HEDLEY_UNREACHABLE(), SIMDE_BFLOAT16_VALUE(0.0)), lane, v);
+  #else
+    simde_bfloat16x8_private v_ = simde_bfloat16x8_to_private(v);
+
+    r = v_.values[lane];
+  #endif
+
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vgetq_lane_bf16
+  #define vgetq_lane_bf16(v, lane) simde_vgetq_lane_bf16((v), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/get_low.h

@@ -415,6 +415,27 @@ simde_vget_low_p64(simde_poly64x2_t a) {
   #define vget_low_p64(a) simde_vget_low_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vget_low_bf16(simde_bfloat16x8_t a) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vget_low_bf16(a);
+  #else
+    simde_bfloat16x4_private r_;
+    simde_bfloat16x8_private a_ = simde_bfloat16x8_to_private(a);
+
+    SIMDE_VECTORIZE
+    for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
+      r_.values[i] = a_.values[i];
+    }
+
+    return simde_bfloat16x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vget_low_bf16
+  #define vget_low_bf16(a) simde_vget_low_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld1.h

@@ -546,6 +546,37 @@ simde_vldrq_p128(simde_poly128_t const ptr[HEDLEY_ARRAY_PARAM(1)]) {
 
 #endif /* !defined(SIMDE_TARGET_NOT_SUPPORT_INT128_TYPE) */
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vld1_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(4)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1_bf16(ptr);
+  #else
+    simde_bfloat16x4_private r_;
+    simde_memcpy(&r_, ptr, sizeof(r_));
+    return simde_bfloat16x4_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_bf16
+  #define vld1_bf16(a) simde_vld1_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vld1q_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(8)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1q_bf16(ptr);
+  #else
+    simde_bfloat16x8_private r_;
+    simde_memcpy(&r_, ptr, sizeof(r_));
+    return simde_bfloat16x8_from_private(r_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_bf16
+  #define vld1q_bf16(a) simde_vld1q_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld1_dup.h

@@ -514,6 +514,33 @@ simde_vld1q_dup_p64(simde_poly64_t const * ptr) {
   #define vld1q_dup_p64(a) simde_vld1q_dup_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vld1_dup_bf16(simde_bfloat16 const * ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1_dup_bf16(ptr);
+  #else
+    return simde_vdup_n_bf16(*ptr);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_dup_bf16
+  #define vld1_dup_bf16(a) simde_vld1_dup_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vld1q_dup_bf16(simde_bfloat16 const * ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1q_dup_bf16(ptr);
+  #else
+    return simde_vdupq_n_bf16(*ptr);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_dup_bf16
+  #define vld1q_dup_bf16(a) simde_vld1q_dup_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld1_lane.h

@@ -488,6 +488,36 @@ simde_vld1q_lane_p64(simde_poly64_t const *ptr, simde_poly64x2_t src,
   #define vld1q_lane_p64(ptr, src, lane) simde_vld1q_lane_p64((ptr), (src), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t simde_vld1_lane_bf16(simde_bfloat16_t const *ptr, simde_bfloat16x4_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16x4_private r = simde_bfloat16x4_to_private(src);
+  r.values[lane] = *ptr;
+  return simde_bfloat16x4_from_private(r);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld1_lane_bf16(ptr, src, lane) vld1_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_lane_bf16
+  #define vld1_lane_bf16(ptr, src, lane) simde_vld1_lane_bf16((ptr), (src), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t simde_vld1q_lane_bf16(simde_bfloat16_t const *ptr, simde_bfloat16x8_t src,
+                                     const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16x8_private r = simde_bfloat16x8_to_private(src);
+  r.values[lane] = *ptr;
+  return simde_bfloat16x8_from_private(r);
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld1q_lane_bf16(ptr, src, lane) vld1q_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_lane_bf16
+  #define vld1q_lane_bf16(ptr, src, lane) simde_vld1q_lane_bf16((ptr), (src), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld1_x2.h

@@ -357,6 +357,25 @@ simde_vld1_p64_x2(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(2)]) {
   #define vld1_p64_x2(a) simde_vld1_p64_x2((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x2_t
+simde_vld1_bf16_x2(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(8)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1_bf16_x2(ptr);
+  #else
+    simde_bfloat16x4_private a_[2];
+    for (size_t i = 0; i < 8; i++) {
+      a_[i / 4].values[i % 4] = ptr[i];
+    }
+    simde_bfloat16x4x2_t s_ = { { simde_bfloat16x4_from_private(a_[0]),
+                                 simde_bfloat16x4_from_private(a_[1]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_bf16_x2
+  #define vld1_bf16_x2(a) simde_vld1_bf16_x2((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld1_x3.h

@@ -372,6 +372,26 @@ simde_vld1_p64_x3(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(3)]) {
   #define vld1_p64_x3(a) simde_vld1_p64_x3((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x3_t
+simde_vld1_bf16_x3(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(12)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1_bf16_x3(ptr);
+  #else
+    simde_bfloat16x4_private a_[3];
+    for (size_t i = 0; i < 12; i++) {
+      a_[i / 4].values[i % 4] = ptr[i];
+    }
+    simde_bfloat16x4x3_t s_ = { { simde_bfloat16x4_from_private(a_[0]),
+                                 simde_bfloat16x4_from_private(a_[1]),
+                                 simde_bfloat16x4_from_private(a_[2]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_bf16_x3
+  #define vld1_bf16_x3(a) simde_vld1_bf16_x3((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld1_x4.h

@@ -387,6 +387,27 @@ simde_vld1_p64_x4(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(4)]) {
   #define vld1_p64_x4(a) simde_vld1_p64_x4((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x4_t
+simde_vld1_bf16_x4(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(16)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1_bf16_x4(ptr);
+  #else
+    simde_bfloat16x4_private a_[4];
+    for (size_t i = 0; i < 16; i++) {
+      a_[i / 4].values[i % 4] = ptr[i];
+    }
+    simde_bfloat16x4x4_t s_ = { { simde_bfloat16x4_from_private(a_[0]),
+                                 simde_bfloat16x4_from_private(a_[1]),
+                                 simde_bfloat16x4_from_private(a_[2]),
+                                 simde_bfloat16x4_from_private(a_[3]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1_bf16_x4
+  #define vld1_bf16_x4(a) simde_vld1_bf16_x4((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld1q_x2.h

@@ -361,6 +361,26 @@ simde_vld1q_p64_x2(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(4)]) {
   #define vld1q_p64_x2(a) simde_vld1q_p64_x2((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x2_t
+simde_vld1q_bf16_x2(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(16)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1q_bf16_x2(ptr);
+  #else
+    simde_bfloat16x8_private a_[2];
+    for (size_t i = 0; i < 16; i++) {
+      a_[i / 8].values[i % 8] = ptr[i];
+    }
+    simde_bfloat16x8x2_t s_ = { { simde_bfloat16x8_from_private(a_[0]),
+                                 simde_bfloat16x8_from_private(a_[1]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_bf16_x2
+  #define vld1q_bf16_x2(a) simde_vld1q_bf16_x2((a))
+#endif
+
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld1q_x3.h

@@ -373,6 +373,26 @@ simde_vld1q_p64_x3(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(3)]) {
   #define vld1q_p64_x3(a) simde_vld1q_p64_x3((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x3_t
+simde_vld1q_bf16_x3(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(24)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1q_bf16_x3(ptr);
+  #else
+    simde_bfloat16x8_private a_[3];
+    for (size_t i = 0; i < 24; i++) {
+      a_[i / 8].values[i % 8] = ptr[i];
+    }
+    simde_bfloat16x8x3_t s_ = { { simde_bfloat16x8_from_private(a_[0]),
+                                 simde_bfloat16x8_from_private(a_[1]),
+                                 simde_bfloat16x8_from_private(a_[2]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_bf16_x3
+  #define vld1q_bf16_x3(a) simde_vld1q_bf16_x3((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld1q_x4.h

@@ -388,6 +388,27 @@ simde_vld1q_p64_x4(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(8)]) {
   #define vld1q_p64_x4(a) simde_vld1q_p64_x4((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x4_t
+simde_vld1q_bf16_x4(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(32)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld1q_bf16_x4(ptr);
+  #else
+    simde_bfloat16x8_private a_[4];
+    for (size_t i = 0; i < 32; i++) {
+      a_[i / 8].values[i % 8] = ptr[i];
+    }
+    simde_bfloat16x8x4_t s_ = { { simde_bfloat16x8_from_private(a_[0]),
+                                 simde_bfloat16x8_from_private(a_[1]),
+                                 simde_bfloat16x8_from_private(a_[2]),
+                                 simde_bfloat16x8_from_private(a_[3]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld1q_bf16_x4
+  #define vld1q_bf16_x4(a) simde_vld1q_bf16_x4((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld2.h

@@ -1013,6 +1013,59 @@ simde_vld2q_p64(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(4)]) {
   #define vld2q_p64(a) simde_vld2q_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x2_t
+simde_vld2_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(8)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld2_bf16(ptr);
+  #else
+    simde_bfloat16x4_private r_[2];
+
+    for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_[0])) ; i++) {
+      for (size_t j = 0 ; j < (sizeof(r_[0].values) / sizeof(r_[0].values[0])) ; j++) {
+        r_[i].values[j] = ptr[i + (j * (sizeof(r_) / sizeof(r_[0])))];
+      }
+    }
+
+    simde_bfloat16x4x2_t r = { {
+      simde_bfloat16x4_from_private(r_[0]),
+      simde_bfloat16x4_from_private(r_[1]),
+    } };
+
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2_bf16
+  #define vld2_bf16(a) simde_vld2_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x2_t
+simde_vld2q_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(16)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld2q_bf16(ptr);
+  #else
+    simde_bfloat16x8_private r_[2];
+
+    for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_[0])); i++) {
+      for (size_t j = 0 ; j < (sizeof(r_[0].values) / sizeof(r_[0].values[0])) ; j++) {
+        r_[i].values[j] = ptr[i + (j * (sizeof(r_) / sizeof(r_[0])))];
+      }
+    }
+
+    simde_bfloat16x8x2_t r = { {
+      simde_bfloat16x8_from_private(r_[0]),
+      simde_bfloat16x8_from_private(r_[1]),
+    } };
+
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2q_bf16
+  #define vld2q_bf16(a) simde_vld2q_bf16((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld2_dup.h

@@ -568,6 +568,43 @@ simde_vld2q_dup_p64(simde_poly64_t const * ptr) {
   #define vld2q_dup_p64(a) simde_vld2q_dup_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x2_t
+simde_vld2_dup_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(2)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld2_dup_bf16(ptr);
+  #else
+    simde_bfloat16x4x2_t r;
+
+    for (size_t i = 0 ; i < 2 ; i++) {
+      r.val[i] = simde_vdup_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2_dup_bf16
+  #define vld2_dup_bf16(a) simde_vld2_dup_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x2_t
+simde_vld2q_dup_bf16(simde_bfloat16 const * ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld2q_dup_bf16(ptr);
+  #else
+    simde_bfloat16x8x2_t r;
+
+    for (size_t i = 0 ; i < 2 ; i++) {
+      r.val[i] = simde_vdupq_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2q_dup_bf16
+  #define vld2q_dup_bf16(a) simde_vld2q_dup_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld2_lane.h

@@ -592,6 +592,45 @@ simde_poly64x2x2_t simde_vld2q_lane_p64(simde_poly64_t const ptr[HEDLEY_ARRAY_PA
   #define vld2q_lane_p64(ptr, src, lane) simde_vld2q_lane_p64((ptr), (src), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x2_t simde_vld2_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(2)], simde_bfloat16x4x2_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16x4x2_t r;
+
+  for (size_t i = 0 ; i < 2 ; i++) {
+    simde_bfloat16x4_private tmp_ = simde_bfloat16x4_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x4_from_private(tmp_);
+  }
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld2_lane_bf16(ptr, src, lane) vld2_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2_lane_bf16
+  #define vld2_lane_bf16(ptr, src, lane) simde_vld2_lane_bf16((ptr), (src), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x2_t simde_vld2q_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(2)], simde_bfloat16x8x2_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16x8x2_t r;
+
+  for (size_t i = 0 ; i < 2 ; i++) {
+    simde_bfloat16x8_private tmp_ = simde_bfloat16x8_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x8_from_private(tmp_);
+  }
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld2q_lane_bf16(ptr, src, lane) vld2q_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld2q_lane_bf16
+  #define vld2q_lane_bf16(ptr, src, lane) simde_vld2q_lane_bf16((ptr), (src), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld3.h

@@ -826,6 +826,61 @@ simde_vld3q_p64(simde_poly64_t const *ptr) {
   #define vld3q_p64(a) simde_vld3q_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x3_t
+simde_vld3_bf16(simde_bfloat16 const *ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld3_bf16(ptr);
+  #else
+    simde_bfloat16x4_private r_[3];
+
+    for (size_t i = 0; i < (sizeof(r_) / sizeof(r_[0])); i++) {
+      for (size_t j = 0 ; j < (sizeof(r_[0].values) / sizeof(r_[0].values[0])) ; j++) {
+        r_[i].values[j] = ptr[i + (j * (sizeof(r_) / sizeof(r_[0])))];
+      }
+    }
+
+    simde_bfloat16x4x3_t r = { {
+      simde_bfloat16x4_from_private(r_[0]),
+      simde_bfloat16x4_from_private(r_[1]),
+      simde_bfloat16x4_from_private(r_[2])
+    } };
+
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3_bf16
+  #define vld3_bf16(a) simde_vld3_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x3_t
+simde_vld3q_bf16(simde_bfloat16 const *ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld3q_bf16(ptr);
+  #else
+    simde_bfloat16x8_private r_[3];
+
+    for (size_t i = 0; i < (sizeof(r_) / sizeof(r_[0])); i++) {
+      for (size_t j = 0 ; j < (sizeof(r_[0].values) / sizeof(r_[0].values[0])) ; j++) {
+        r_[i].values[j] = ptr[i + (j * (sizeof(r_) / sizeof(r_[0])))];
+      }
+    }
+
+    simde_bfloat16x8x3_t r = { {
+      simde_bfloat16x8_from_private(r_[0]),
+      simde_bfloat16x8_from_private(r_[1]),
+      simde_bfloat16x8_from_private(r_[2])
+    } };
+
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3q_bf16
+  #define vld3q_bf16(a) simde_vld3q_bf16((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld3_dup.h

@@ -566,6 +566,43 @@ simde_vld3q_dup_p64(simde_poly64_t const * ptr) {
   #define vld3q_dup_p64(a) simde_vld3q_dup_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x3_t
+simde_vld3_dup_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(2)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld3_dup_bf16(ptr);
+  #else
+    simde_bfloat16x4x3_t r;
+
+    for (size_t i = 0 ; i < 3 ; i++) {
+      r.val[i] = simde_vdup_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3_dup_bf16
+  #define vld3_dup_bf16(a) simde_vld3_dup_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x3_t
+simde_vld3q_dup_bf16(simde_bfloat16 const * ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld3q_dup_bf16(ptr);
+  #else
+    simde_bfloat16x8x3_t r;
+
+    for (size_t i = 0 ; i < 3 ; i++) {
+      r.val[i] = simde_vdupq_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3q_dup_bf16
+  #define vld3q_dup_bf16(a) simde_vld3q_dup_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld3_lane.h

@@ -592,6 +592,45 @@ simde_poly64x2x3_t simde_vld3q_lane_p64(simde_poly64_t const ptr[HEDLEY_ARRAY_PA
   #define vld3q_lane_p64(ptr, src, lane) simde_vld3q_lane_p64((ptr), (src), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x3_t simde_vld3_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(3)], simde_bfloat16x4x3_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16x4x3_t r;
+
+  for (size_t i = 0 ; i < 3 ; i++) {
+    simde_bfloat16x4_private tmp_ = simde_bfloat16x4_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x4_from_private(tmp_);
+  }
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld3_lane_bf16(ptr, src, lane) vld3_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3_lane_bf16
+  #define vld3_lane_bf16(ptr, src, lane) simde_vld3_lane_bf16((ptr), (src), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x3_t simde_vld3q_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(3)], simde_bfloat16x8x3_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16x8x3_t r;
+
+  for (size_t i = 0 ; i < 3 ; i++) {
+    simde_bfloat16x8_private tmp_ = simde_bfloat16x8_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x8_from_private(tmp_);
+  }
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld3q_lane_bf16(ptr, src, lane) vld3q_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld3q_lane_bf16
+  #define vld3q_lane_bf16(ptr, src, lane) simde_vld3q_lane_bf16((ptr), (src), (lane))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld4.h

@@ -638,6 +638,45 @@ simde_vld4q_p64(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(8)]) {
   #define vld4q_p64(a) simde_vld4q_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x4_t
+simde_vld4_bf16(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(16)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld4_bf16(ptr);
+  #else
+    simde_bfloat16x4_private a_[4];
+    for (size_t i = 0; i < (sizeof(simde_bfloat16x4_t) / sizeof(*ptr)) * 4 ; i++) {
+      a_[i % 4].values[i / 4] = ptr[i];
+    }
+    simde_bfloat16x4x4_t s_ = { { simde_bfloat16x4_from_private(a_[0]), simde_bfloat16x4_from_private(a_[1]),
+                                 simde_bfloat16x4_from_private(a_[2]), simde_bfloat16x4_from_private(a_[3]) } };
+    return (s_);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4_bf16
+  #define vld4_bf16(a) simde_vld4_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x4_t
+simde_vld4q_bf16(simde_bfloat16 const ptr[HEDLEY_ARRAY_PARAM(32)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld4q_bf16(ptr);
+  #else
+    simde_bfloat16x8_private a_[4];
+    for (size_t i = 0; i < (sizeof(simde_bfloat16x8_t) / sizeof(*ptr)) * 4 ; i++) {
+      a_[i % 4].values[i / 4] = ptr[i];
+    }
+    simde_bfloat16x8x4_t s_ = { { simde_bfloat16x8_from_private(a_[0]), simde_bfloat16x8_from_private(a_[1]),
+                                 simde_bfloat16x8_from_private(a_[2]), simde_bfloat16x8_from_private(a_[3]) } };
+    return s_;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4q_bf16
+  #define vld4q_bf16(a) simde_vld4q_bf16((a))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/ld4_dup.h

@@ -566,6 +566,43 @@ simde_vld4q_dup_p64(simde_poly64_t const * ptr) {
   #define vld4q_dup_p64(a) simde_vld4q_dup_p64((a))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x4_t
+simde_vld4_dup_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(2)]) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld4_dup_bf16(ptr);
+  #else
+    simde_bfloat16x4x4_t r;
+
+    for (size_t i = 0 ; i < 4 ; i++) {
+      r.val[i] = simde_vdup_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4_dup_bf16
+  #define vld4_dup_bf16(a) simde_vld4_dup_bf16((a))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x4_t
+simde_vld4q_dup_bf16(simde_bfloat16 const * ptr) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    return vld4q_dup_bf16(ptr);
+  #else
+    simde_bfloat16x8x4_t r;
+
+    for (size_t i = 0 ; i < 4 ; i++) {
+      r.val[i] = simde_vdupq_n_bf16(ptr[i]);
+    }
+    return r;
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4q_dup_bf16
+  #define vld4q_dup_bf16(a) simde_vld4q_dup_bf16((a))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/ld4_lane.h

@@ -773,6 +773,49 @@ simde_vld4q_lane_p64(simde_poly64_t const ptr[HEDLEY_ARRAY_PARAM(4)], simde_poly
   #define vld4q_lane_p64(ptr, src, lane) simde_vld4q_lane_p64((ptr), (src), (lane))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4x4_t
+simde_vld4_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(4)], simde_bfloat16x4x4_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16x4x4_t r;
+
+  for (size_t i = 0 ; i < 4 ; i++) {
+    simde_bfloat16x4_private tmp_ = simde_bfloat16x4_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x4_from_private(tmp_);
+  }
+
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld4_lane_bf16(ptr, src, lane) vld4_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4_lane_bf16
+  #define vld4_lane_bf16(ptr, src, lane) simde_vld4_lane_bf16((ptr), (src), (lane))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8x4_t
+simde_vld4q_lane_bf16(simde_bfloat16_t const ptr[HEDLEY_ARRAY_PARAM(4)], simde_bfloat16x8x4_t src, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16x8x4_t r;
+
+  for (size_t i = 0 ; i < 4 ; i++) {
+    simde_bfloat16x8_private tmp_ = simde_bfloat16x8_to_private(src.val[i]);
+    tmp_.values[lane] = ptr[i];
+    r.val[i] = simde_bfloat16x8_from_private(tmp_);
+  }
+
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  #define simde_vld4q_lane_bf16(ptr, src, lane) vld4q_lane_bf16(ptr, src, lane)
+#endif
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vld4q_lane_bf16
+  #define vld4q_lane_bf16(ptr, src, lane) simde_vld4q_lane_bf16((ptr), (src), (lane))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/mmlaq.h

@@ -118,6 +118,34 @@ simde_vusmmlaq_s32(simde_int32x4_t r, simde_uint8x16_t a, simde_int8x16_t b) {
   #define vusmmlaq_s32(r, a, b) simde_vusmmlaq_s32((r), (a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_float32x4_t
+simde_vbfmmlaq_f32(simde_float32x4_t r, simde_bfloat16x8_t a, simde_bfloat16x8_t b) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(__ARM_FEATURE_MATMUL_INT8) && \
+      defined(SIMDE_ARM_NEON_BF16)
+    return vbfmmlaq_f32(r, a, b);
+  #else
+    simde_bfloat16x8_private
+      a_ = simde_bfloat16x8_to_private(a),
+      b_ = simde_bfloat16x8_to_private(b);
+    simde_float32x4_private
+      r_ = simde_float32x4_to_private(r),
+      ret;
+
+    for (size_t k = 0 ; k < (sizeof(ret.values) / sizeof(ret.values[0])) ; k++) {
+      ret.values[k] = r_.values[k];
+      for (size_t i = 0 ; i < (sizeof(a_.values) / sizeof(a_.values[0]) / 2) ; i++) {
+         ret.values[k] += simde_bfloat16_to_float32(a_.values[(k/2)*4+i]) *
+                          simde_bfloat16_to_float32(b_.values[(k%2)*4+i]);
+      }
+    }
+    return simde_float32x4_from_private(ret);
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vbfmmlaq_f32
+  #define vbfmmlaq_f32(r, a, b) simde_vbfmmlaq_f32((r), (a), (b))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/set_lane.h

@@ -563,6 +563,43 @@ simde_vsetq_lane_p64(simde_poly64_t a, simde_poly64x2_t v, const int lane)
   #define vsetq_lane_p64(a, b, c) simde_vsetq_lane_p64((a), (b), (c))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x4_t
+simde_vset_lane_bf16(simde_bfloat16_t a, simde_bfloat16x4_t v, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  simde_bfloat16x4_t r;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_(vset_lane_bf16, r, (HEDLEY_UNREACHABLE(), v), lane, a, v);
+  #else
+    simde_bfloat16x4_private v_ = simde_bfloat16x4_to_private(v);
+    v_.values[lane] = a;
+    r = simde_bfloat16x4_from_private(v_);
+  #endif
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vset_lane_bf16
+  #define vset_lane_bf16(a, b, c) simde_vset_lane_bf16((a), (b), (c))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+simde_bfloat16x8_t
+simde_vsetq_lane_bf16(simde_bfloat16_t a, simde_bfloat16x8_t v, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  simde_bfloat16x8_t r;
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_(vsetq_lane_bf16, r, (HEDLEY_UNREACHABLE(), v), lane, a, v);
+  #else
+    simde_bfloat16x8_private v_ = simde_bfloat16x8_to_private(v);
+    v_.values[lane] = a;
+    r = simde_bfloat16x8_from_private(v_);
+  #endif
+  return r;
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vsetq_lane_bf16
+  #define vsetq_lane_bf16(a, b, c) simde_vsetq_lane_bf16((a), (b), (c))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/st1.h

@@ -525,6 +525,35 @@ simde_vstrq_p128(simde_poly128_t ptr[HEDLEY_ARRAY_PARAM(1)], simde_poly128_t val
 #endif
 #endif /* !defined(SIMDE_TARGET_NOT_SUPPORT_INT128_TYPE) */
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_bfloat16x4_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1_bf16(ptr, val);
+  #else
+    simde_bfloat16x4_private val_ = simde_bfloat16x4_to_private(val);
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1_bf16
+  #define vst1_bf16(a, b) simde_vst1_bf16((a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1q_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(8)], simde_bfloat16x8_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1q_bf16(ptr, val);
+  #else
+    simde_bfloat16x8_private val_ = simde_bfloat16x8_to_private(val);
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1q_bf16
+  #define vst1q_bf16(a, b) simde_vst1q_bf16((a), (b))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/st1_lane.h

@@ -486,6 +486,37 @@ simde_vst1q_lane_p64(simde_poly64_t *ptr, simde_poly64x2_t val, const int lane)
   #define vst1q_lane_p64(a, b, c) simde_vst1q_lane_p64((a), (b), (c))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1_lane_bf16(simde_bfloat16_t *ptr, simde_bfloat16x4_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_NO_RESULT_(vst1_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x4_private val_ = simde_bfloat16x4_to_private(val);
+    *ptr = val_.values[lane];
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1_lane_bf16
+  #define vst1_lane_bf16(a, b, c) simde_vst1_lane_bf16((a), (b), (c))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1q_lane_bf16(simde_bfloat16_t *ptr, simde_bfloat16x8_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_NO_RESULT_(vst1q_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x8_private val_ = simde_bfloat16x8_to_private(val);
+    *ptr = val_.values[lane];
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1q_lane_bf16
+  #define vst1q_lane_bf16(a, b, c) simde_vst1q_lane_bf16((a), (b), (c))
+#endif
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/st1_x2.h

@@ -262,6 +262,23 @@ simde_vst1_p64_x2(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(2)], simde_poly64x1x2_t
   #define vst1_p64_x2(a, b) simde_vst1_p64_x2((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1_bf16_x2(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(8)], simde_bfloat16x4x2_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1_bf16_x2(ptr, val);
+  #else
+    simde_bfloat16x4_private val_[2];
+    for (size_t i = 0; i < 2; i++) {
+      val_[i] = simde_bfloat16x4_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1_bf16_x2
+  #define vst1_bf16_x2(a, b) simde_vst1_bf16_x2((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st1_x3.h

@@ -272,6 +272,23 @@ simde_vst1_p64_x3(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(3)], simde_poly64x1x3_t
   #define vst1_p64_x3(a, b) simde_vst1_p64_x3((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1_bf16_x3(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(12)], simde_bfloat16x4x3_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1_bf16_x3(ptr, val);
+  #else
+    simde_bfloat16x4_private val_[3];
+    for (size_t i = 0; i < 3; i++) {
+      val_[i] = simde_bfloat16x4_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1_bf16_x3
+  #define vst1_bf16_x3(a, b) simde_vst1_bf16_x3((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st1_x4.h

@@ -282,6 +282,23 @@ simde_vst1_p64_x4(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_poly64x1x4_t
   #define vst1_p64_x4(a, b) simde_vst1_p64_x4((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1_bf16_x4(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(16)], simde_bfloat16x4x4_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1_bf16_x4(ptr, val);
+  #else
+    simde_bfloat16x4_private val_[4];
+    for (size_t i = 0; i < 4; i++) {
+      val_[i] = simde_bfloat16x4_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1_bf16_x4
+  #define vst1_bf16_x4(a, b) simde_vst1_bf16_x4((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st1q_x2.h

@@ -260,6 +260,23 @@ simde_vst1q_p64_x2(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_poly64x2x2_t
   #define vst1q_p64_x2(a, b) simde_vst1q_p64_x2((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1q_bf16_x2(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(16)], simde_bfloat16x8x2_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1q_bf16_x2(ptr, val);
+  #else
+    simde_bfloat16x8_private val_[2];
+    for (size_t i = 0; i < 2; i++) {
+      val_[i] = simde_bfloat16x8_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1q_bf16_x2
+  #define vst1q_bf16_x2(a, b) simde_vst1q_bf16_x2((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st1q_x3.h

@@ -270,6 +270,23 @@ simde_vst1q_p64_x3(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(6)], simde_poly64x2x3_t
   #define vst1q_p64_x3(a, b) simde_vst1q_p64_x3((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1q_bf16_x3(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(24)], simde_bfloat16x8x3_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1q_bf16_x3(ptr, val);
+  #else
+    simde_bfloat16x8_private val_[3];
+    for (size_t i = 0; i < 3; i++) {
+      val_[i] = simde_bfloat16x8_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1q_bf16_x3
+  #define vst1q_bf16_x3(a, b) simde_vst1q_bf16_x3((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st1q_x4.h

@@ -282,6 +282,23 @@ simde_vst1q_p64_x4(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(8)], simde_poly64x2x4_t
   #define vst1q_p64_x4(a, b) simde_vst1q_p64_x4((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst1q_bf16_x4(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(32)], simde_bfloat16x8x4_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst1q_bf16_x4(ptr, val);
+  #else
+    simde_bfloat16x8_private val_[4];
+    for (size_t i = 0; i < 4; i++) {
+      val_[i] = simde_bfloat16x8_to_private(val.val[i]);
+    }
+    simde_memcpy(ptr, &val_, sizeof(val_));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst1q_bf16_x4
+  #define vst1q_bf16_x4(a, b) simde_vst1q_bf16_x4((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st2.h

@@ -566,6 +566,45 @@ simde_vst2q_p64(simde_poly64_t *ptr, simde_poly64x2x2_t val) {
   #define vst2q_p64(a, b) simde_vst2q_p64((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst2_bf16(simde_bfloat16_t *ptr, simde_bfloat16x4x2_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst2_bf16(ptr, val);
+  #else
+    simde_bfloat16_t buf[8];
+    simde_bfloat16x4_private a_[2] = {simde_bfloat16x4_to_private(val.val[0]),
+                                     simde_bfloat16x4_to_private(val.val[1])};
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 2 ; i++) {
+      buf[i] = a_[i % 2].values[i / 2];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst2_bf16
+  #define vst2_bf16(a, b) simde_vst2_bf16((a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst2q_bf16(simde_bfloat16_t *ptr, simde_bfloat16x8x2_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst2q_bf16(ptr, val);
+  #else
+    simde_bfloat16_t buf[16];
+    simde_bfloat16x8_private a_[2] = {simde_bfloat16x8_to_private(val.val[0]),
+                                     simde_bfloat16x8_to_private(val.val[1])};
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 2 ; i++) {
+      buf[i] = a_[i % 2].values[i / 2];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst2q_bf16
+  #define vst2q_bf16(a, b) simde_vst2q_bf16((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st2_lane.h

@@ -572,6 +572,43 @@ simde_vst2q_lane_p64(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(2)], simde_poly64x2x2
   #define vst2q_lane_p64(a, b, c) simde_vst2q_lane_p64((a), (b), (c))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst2_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(2)], simde_bfloat16x4x2_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_NO_RESULT_(vst2_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x4_private r;
+    for (size_t i = 0 ; i < 2 ; i ++) {
+      r = simde_bfloat16x4_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst2_lane_bf16
+  #define vst2_lane_bf16(a, b, c) simde_vst2_lane_bf16((a), (b), (c))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst2q_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(2)], simde_bfloat16x8x2_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_NO_RESULT_(vst2q_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x8_private r;
+    for (size_t i = 0 ; i < 2 ; i++) {
+      r = simde_bfloat16x8_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst2q_lane_bf16
+  #define vst2q_lane_bf16(a, b, c) simde_vst2q_lane_bf16((a), (b), (c))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st3.h

@@ -937,6 +937,47 @@ simde_vst3q_p64(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(6)], simde_poly64x2x3_t va
   #define vst3q_p64(a, b) simde_vst3q_p64((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst3_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(12)], simde_bfloat16x4x3_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst3_bf16(ptr, val);
+  #else
+    simde_bfloat16x4_private a[3] = { simde_bfloat16x4_to_private(val.val[0]),
+                                      simde_bfloat16x4_to_private(val.val[1]),
+                                      simde_bfloat16x4_to_private(val.val[2]) };
+    simde_bfloat16_t buf[12];
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 3 ; i++) {
+      buf[i] = a[i % 3].values[i / 3];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst3_bf16
+  #define vst3_bf16(a, b) simde_vst3_bf16((a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst3q_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(24)], simde_bfloat16x8x3_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst3q_bf16(ptr, val);
+  #else
+    simde_bfloat16x8_private a_[3] = { simde_bfloat16x8_to_private(val.val[0]),
+                                      simde_bfloat16x8_to_private(val.val[1]),
+                                      simde_bfloat16x8_to_private(val.val[2]) };
+    simde_bfloat16_t buf[24];
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 3 ; i++) {
+      buf[i] = a_[i % 3].values[i / 3];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst3q_bf16
+  #define vst3q_bf16(a, b) simde_vst3q_bf16((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st3_lane.h

@@ -572,6 +572,43 @@ simde_vst3q_lane_p64(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(3)], simde_poly64x2x3
   #define vst3q_lane_p64(a, b, c) simde_vst3q_lane_p64((a), (b), (c))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst3_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(3)], simde_bfloat16x4x3_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_NO_RESULT_(vst3_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x4_private r;
+    for (size_t i = 0 ; i < 3 ; i++) {
+      r = simde_bfloat16x4_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst3_lane_bf16
+  #define vst3_lane_bf16(a, b, c) simde_vst3_lane_bf16((a), (b), (c))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst3q_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(3)], simde_bfloat16x8x3_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_NO_RESULT_(vst3q_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x8_private r;
+    for (size_t i = 0 ; i < 3 ; i++) {
+      r = simde_bfloat16x8_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst3q_lane_bf16
+  #define vst3q_lane_bf16(a, b, c) simde_vst3q_lane_bf16((a), (b), (c))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st4.h

@@ -598,6 +598,45 @@ simde_vst4q_p64(simde_poly64_t *ptr, simde_poly64x2x4_t val) {
   #define vst4q_p64(a, b) simde_vst4q_p64((a), (b))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst4_bf16(simde_bfloat16_t *ptr, simde_bfloat16x4x4_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst4_bf16(ptr, val);
+  #else
+    simde_bfloat16_t buf[16];
+    simde_bfloat16x4_private a_[4] = { simde_bfloat16x4_to_private(val.val[0]), simde_bfloat16x4_to_private(val.val[1]),
+                                      simde_bfloat16x4_to_private(val.val[2]), simde_bfloat16x4_to_private(val.val[3]) };
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 4 ; i++) {
+      buf[i] = a_[i % 4].values[i / 4];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst4_bf16
+  #define vst4_bf16(a, b) simde_vst4_bf16((a), (b))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst4q_bf16(simde_bfloat16_t *ptr, simde_bfloat16x8x4_t val) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    vst4q_bf16(ptr, val);
+  #else
+    simde_bfloat16_t buf[32];
+    simde_bfloat16x8_private a_[4] = { simde_bfloat16x8_to_private(val.val[0]), simde_bfloat16x8_to_private(val.val[1]),
+                                      simde_bfloat16x8_to_private(val.val[2]), simde_bfloat16x8_to_private(val.val[3]) };
+    for (size_t i = 0; i < (sizeof(val.val[0]) / sizeof(*ptr)) * 4 ; i++) {
+      buf[i] = a_[i % 4].values[i / 4];
+    }
+    simde_memcpy(ptr, buf, sizeof(buf));
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst4q_bf16
+  #define vst4q_bf16(a, b) simde_vst4q_bf16((a), (b))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/st4_lane.h

@@ -573,6 +573,43 @@ simde_vst4q_lane_p64(simde_poly64_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_poly64x2x4
   #define vst4q_lane_p64(a, b, c) simde_vst4q_lane_p64((a), (b), (c))
 #endif
 
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst4_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_bfloat16x4x4_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_4_NO_RESULT_(vst4_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x4_private r;
+    for (size_t i = 0 ; i < 4 ; i++) {
+      r = simde_bfloat16x4_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst4_lane_bf16
+  #define vst4_lane_bf16(a, b, c) simde_vst4_lane_bf16((a), (b), (c))
+#endif
+
+SIMDE_FUNCTION_ATTRIBUTES
+void
+simde_vst4q_lane_bf16(simde_bfloat16_t ptr[HEDLEY_ARRAY_PARAM(4)], simde_bfloat16x8x4_t val, const int lane)
+    SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
+  #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+    SIMDE_CONSTIFY_8_NO_RESULT_(vst4q_lane_bf16, HEDLEY_UNREACHABLE(), lane, ptr, val);
+  #else
+    simde_bfloat16x8_private r;
+    for (size_t i = 0 ; i < 4 ; i++) {
+      r = simde_bfloat16x8_to_private(val.val[i]);
+      ptr[i] = r.values[lane];
+    }
+  #endif
+}
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
+  #undef vst4q_lane_bf16
+  #define vst4q_lane_bf16(a, b, c) simde_vst4q_lane_bf16((a), (b), (c))
+#endif
 
 #endif /* !defined(SIMDE_BUG_INTEL_857088) */
 

arm/neon/types.h

@@ -30,6 +30,7 @@
 
 #include "../../simde-common.h"
 #include "../../simde-f16.h"
+#include "../../simde-bf16.h"
 
 HEDLEY_DIAGNOSTIC_PUSH
 SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
@@ -341,6 +342,22 @@ typedef union {
   SIMDE_ARM_NEON_DECLARE_VECTOR(simde_poly64, values, 16);
 } simde_poly64x2_private;
 
+typedef union {
+  #if SIMDE_BFLOAT16_API == SIMDE_BFLOAT16_API_BF16
+    SIMDE_ARM_NEON_DECLARE_VECTOR(simde_bfloat16, values, 8);
+  #else
+    simde_bfloat16 values[4];
+  #endif
+} simde_bfloat16x4_private;
+
+typedef union {
+  #if SIMDE_BFLOAT16_API == SIMDE_BFLOAT16_API_BF16
+    SIMDE_ARM_NEON_DECLARE_VECTOR(simde_bfloat16, values, 16);
+  #else
+    simde_bfloat16 values[8];
+  #endif
+} simde_bfloat16x8_private;
+
 #if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
   typedef     float32_t     simde_float32_t;
   typedef       poly8_t       simde_poly8_t;
@@ -456,6 +473,20 @@ typedef union {
     #define SIMDE_ARM_NEON_NEED_PORTABLE_F16
   #endif
 
+  #if defined(SIMDE_ARM_NEON_BF16)
+    typedef     bfloat16_t     simde_bfloat16_t;
+    typedef   bfloat16x4_t   simde_bfloat16x4_t;
+    typedef bfloat16x4x2_t simde_bfloat16x4x2_t;
+    typedef bfloat16x4x3_t simde_bfloat16x4x3_t;
+    typedef bfloat16x4x4_t simde_bfloat16x4x4_t;
+    typedef   bfloat16x8_t   simde_bfloat16x8_t;
+    typedef bfloat16x8x2_t simde_bfloat16x8x2_t;
+    typedef bfloat16x8x3_t simde_bfloat16x8x3_t;
+    typedef bfloat16x8x4_t simde_bfloat16x8x4_t;
+  #else
+    #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
+  #endif
+
   #if defined(SIMDE_ARM_NEON_A32V8_NATIVE)
     typedef      poly64_t      simde_poly64_t;
     typedef    poly64x1_t    simde_poly64x1_t;
@@ -501,6 +532,7 @@ typedef union {
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_64_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_128_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_VXN
+  #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
 
   #define SIMDE_ARM_NEON_NEED_PORTABLE_VXN
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F64X1XN
@@ -564,6 +596,7 @@ typedef union {
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_64_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_128_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_VXN
+  #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
 
   #define SIMDE_ARM_NEON_NEED_PORTABLE_64BIT
 
@@ -590,6 +623,7 @@ typedef union {
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_64_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_128_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_VXN
+  #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
 
   #define SIMDE_ARM_NEON_NEED_PORTABLE_64BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F64X1XN
@@ -667,6 +701,7 @@ typedef union {
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_64_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_128_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_VXN
+  #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
   #define SIMDE_ARM_NEON_NEED_PORTABLE_VXN
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F64X1XN
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F64X2XN
@@ -675,6 +710,7 @@ typedef union {
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_64_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_128_BIT
   #define SIMDE_ARM_NEON_NEED_PORTABLE_POLY_VXN
+  #define SIMDE_ARM_NEON_NEED_PORTABLE_BF16
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F16
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F32
   #define SIMDE_ARM_NEON_NEED_PORTABLE_F64
@@ -765,6 +801,35 @@ typedef union {
   } simde_poly16x8x4_t;
 #endif
 
+#if defined(SIMDE_ARM_NEON_NEED_PORTABLE_BF16)
+  typedef  simde_bfloat16   simde_bfloat16_t;
+  typedef simde_bfloat16x4_private simde_bfloat16x4_t;
+  typedef simde_bfloat16x8_private simde_bfloat16x8_t;
+  typedef struct simde_bfloat16x4x2_t {
+    simde_bfloat16x4_t val[2];
+  } simde_bfloat16x4x2_t;
+
+  typedef struct simde_bfloat16x8x2_t {
+    simde_bfloat16x8_t val[2];
+  } simde_bfloat16x8x2_t;
+
+  typedef struct simde_bfloat16x4x3_t {
+    simde_bfloat16x4_t val[3];
+  } simde_bfloat16x4x3_t;
+
+  typedef struct simde_bfloat16x8x3_t {
+    simde_bfloat16x8_t val[3];
+  } simde_bfloat16x8x3_t;
+
+  typedef struct simde_bfloat16x4x4_t {
+    simde_bfloat16x4_t val[4];
+  } simde_bfloat16x4x4_t;
+
+  typedef struct simde_bfloat16x8x4_t {
+    simde_bfloat16x8_t val[4];
+  } simde_bfloat16x8x4_t;
+#endif
+
 #if defined(SIMDE_ARM_NEON_NEED_PORTABLE_I8X8) || defined(SIMDE_ARM_NEON_NEED_PORTABLE_64BIT)
   typedef simde_int8x8_private simde_int8x8_t;
 #endif
@@ -1271,6 +1336,7 @@ SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(float64x1)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(poly8x8)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(poly16x4)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(poly64x1)
+SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(bfloat16x4)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(int8x16)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(int16x8)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(int32x4)
@@ -1285,6 +1351,7 @@ SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(poly64x2)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(float16x8)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(float32x4)
 SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(float64x2)
+SIMDE_ARM_NEON_TYPE_DEFINE_CONVERSIONS_(bfloat16x8)
 
 SIMDE_END_DECLS_
 HEDLEY_DIAGNOSTIC_POP

arm/neon/usdot.h

@@ -56,7 +56,7 @@ simde_vusdot_s32(simde_int32x2_t r, simde_uint8x8_t a, simde_int8x8_t b) {
     return simde_vadd_s32(r, simde_int32x2_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_MATMUL_INT8))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vusdot_s32
   #define vusdot_s32(r, a, b) simde_vusdot_s32((r), (a), (b))
 #endif
@@ -82,7 +82,7 @@ simde_vusdotq_s32(simde_int32x4_t r, simde_uint8x16_t a, simde_int8x16_t b) {
     return simde_vaddq_s32(r, simde_int32x4_from_private(r_));
   #endif
 }
-#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(__ARM_FEATURE_MATMUL_INT8))
+#if defined(SIMDE_ARM_NEON_A32V8_ENABLE_NATIVE_ALIASES)
   #undef vusdotq_s32
   #define vusdotq_s32(r, a, b) simde_vusdotq_s32((r), (a), (b))
 #endif

simde-arch.h

@@ -592,6 +592,11 @@
 #  define SIMDE_ARCH_ARM_NEON_FP16
 #endif
 
+/* Availability of 16-bit brain floating-point arithmetic intrinsics */
+#if defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)
+#  define SIMDE_ARCH_ARM_NEON_BF16
+#endif
+
 /* LoongArch
    <https://en.wikipedia.org/wiki/Loongson#LoongArch> */
 #if defined(__loongarch32)

simde-bf16.h

+/* SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Copyright:
+ *   2023      Yi-Yen Chung <eric681@andestech.com> (Copyright owned by Andes Technology)
+ */
+
+#include "hedley.h"
+#include "simde-common.h"
+#include "simde-detect-clang.h"
+
+#if !defined(SIMDE_BFLOAT16_H)
+#define SIMDE_BFLOAT16_H
+
+HEDLEY_DIAGNOSTIC_PUSH
+SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
+SIMDE_BEGIN_DECLS_
+
+/* This implementations is based upon simde-f16.h */
+
+/* Portable version which should work on pretty much any compiler.
+ * Obviously you can't rely on compiler support for things like
+ * conversion to/from 32-bit floats, so make sure you always use the
+ * functions and macros in this file!
+ */
+#define SIMDE_BFLOAT16_API_PORTABLE 1
+
+#define SIMDE_BFLOAT16_API_BF16 2
+
+#if !defined(SIMDE_BFLOAT16_API)
+  #if defined(SIMDE_ARM_NEON_BF16)
+    #define SIMDE_BFLOAT16_API SIMDE_BFLOAT16_API_BF16
+  #else
+    #define SIMDE_BFLOAT16_API SIMDE_BFLOAT16_API_PORTABLE
+  #endif
+#endif
+
+#if SIMDE_BFLOAT16_API == SIMDE_BFLOAT16_API_BF16
+  #include <arm_bf16.h>
+  typedef __bf16 simde_bfloat16;
+#elif SIMDE_BFLOAT16_API == SIMDE_BFLOAT16_API_PORTABLE
+  typedef struct { uint16_t value; } simde_bfloat16;
+#else
+  #error No 16-bit floating point API.
+#endif
+
+/* Conversion -- convert between single-precision and brain half-precision
+ * floats. */
+static HEDLEY_ALWAYS_INLINE HEDLEY_CONST
+simde_bfloat16
+simde_bfloat16_from_float32 (simde_float32 value) {
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  return vcvth_bf16_f32(value);
+#else
+  simde_bfloat16 res;
+  char* src = HEDLEY_REINTERPRET_CAST(char*, &value);
+  // rounding to nearest bfloat16
+  // If the 17th bit of value is 1, set the rounding to 1.
+  uint8_t rounding = 0;
+
+  #if SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE
+    if (src[1] & UINT8_C(0x80)) rounding = 1;
+    src[2] = HEDLEY_STATIC_CAST(char, (HEDLEY_STATIC_CAST(uint8_t, src[2]) + rounding));
+    simde_memcpy(&res, src+2, sizeof(res));
+  #else
+    if (src[2] & UINT8_C(0x80)) rounding = 1;
+    src[1] = HEDLEY_STATIC_CAST(char, (HEDLEY_STATIC_CAST(uint8_t, src[1]) + rounding));
+    simde_memcpy(&res, src, sizeof(res));
+  #endif
+
+  return res;
+#endif
+}
+
+static HEDLEY_ALWAYS_INLINE HEDLEY_CONST
+simde_float32
+simde_bfloat16_to_float32 (simde_bfloat16 value) {
+#if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARM_NEON_BF16)
+  return vcvtah_f32_bf16(value);
+#else
+  simde_float32 res = 0.0;
+  char* _res = HEDLEY_REINTERPRET_CAST(char*, &res);
+
+  #if SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE
+    simde_memcpy(_res+2, &value, sizeof(value));
+  #else
+    simde_memcpy(_res, &value, sizeof(value));
+  #endif
+
+  return res;
+#endif
+}
+
+SIMDE_DEFINE_CONVERSION_FUNCTION_(simde_uint16_as_bfloat16, simde_bfloat16,      uint16_t)
+
+#define SIMDE_NANBF simde_uint16_as_bfloat16(0xFFC1) // a quiet Not-a-Number
+#define SIMDE_INFINITYBF simde_uint16_as_bfloat16(0x7F80)
+#define SIMDE_NINFINITYBF simde_uint16_as_bfloat16(0xFF80)
+
+#define SIMDE_BFLOAT16_VALUE(value) simde_bfloat16_from_float32(SIMDE_FLOAT32_C(value))
+
+#if !defined(simde_isinfbf) && defined(simde_math_isinff)
+  #define simde_isinfbf(a) simde_math_isinff(simde_bfloat16_to_float32(a))
+#endif
+#if !defined(simde_isnanbf) && defined(simde_math_isnanf)
+  #define simde_isnanbf(a) simde_math_isnanf(simde_bfloat16_to_float32(a))
+#endif
+
+SIMDE_END_DECLS_
+HEDLEY_DIAGNOSTIC_POP
+
+#endif /* !defined(SIMDE_BFLOAT16_H) */

simde-features.h

@@ -720,6 +720,10 @@
   #define SIMDE_ARM_NEON_FP16
 #endif
 
+#if defined(SIMDE_ARCH_ARM_NEON_BF16)
+  #define SIMDE_ARM_NEON_BF16
+#endif
+
 #if !defined(SIMDE_LOONGARCH_LASX_NATIVE) && !defined(SIMDE_LOONGARCH_LASX_NO_NATIVE) && !defined(SIMDE_NO_NATIVE)
   #if defined(SIMDE_ARCH_LOONGARCH_LASX)
     #define SIMDE_LOONGARCH_LASX_NATIVE