Skip to content

S
simde
Commit 69c9cd5c authored by Ryo Suzuki's avatar Ryo Suzuki Committed by GitHub
Browse files
Options

arm neon qdmlal: fix saturation (#1194) 

* Fix qdmlal instructions

qdmlal instructions were implemented without saturation.
This has been fixed by utilising existing SIMDe saturating mult and add instructions.
Unit tests have been updated to test for all possible saturation cases.

- Fix qdmlal, qdmlal_n, qdmlal_lane,  qdmlal_high, qdmlal_high_n and qdmlal_high_lane
- Update unit tests for qdmlal, qdmlal_n, qdmlal_lane, qdmlal_high, qdmlal_high_n, qdmala_high_lane

Change-Id: I8d0d8cfba3f8d5203f2028efbe74b00c51485c61

* arm neon qdmlal_high_lane: unroll SIMDE_CONSTIFY for testing macro implemented functions with MSVC

---------
Co-authored-by: default avatarMichael R. Crusoe <michael.crusoe@gmail.com>
parent 34136823
Hide whitespace changes
Inline Side-by-side
Showing with 25 additions and 106 deletions
arm/neon/qdmlal.h
View file @ 69c9cd5c
......@@ -27,12 +27,9 @@
#if !defined(SIMDE_ARM_NEON_QDMLAL_H)
#define SIMDE_ARM_NEON_QDMLAL_H
#include "add.h"
#include "mul.h"
#include "mul_n.h"
#include "movl.h"
#include "qadd.h"
#include "types.h"
#include "qadd.h"
#include "qdmull.h"
HEDLEY_DIAGNOSTIC_PUSH
SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
......@@ -44,7 +41,7 @@ simde_vqdmlalh_s16(int32_t a, int16_t b, int16_t c) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlalh_s16(a, b, c);
#else
return HEDLEY_STATIC_CAST(int32_t, b) * HEDLEY_STATIC_CAST(int32_t, c) * 2 + a;
return simde_vqadds_s32(a, simde_vqdmullh_s16(b, c));
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......@@ -58,7 +55,7 @@ simde_vqdmlals_s32(int64_t a, int32_t b, int32_t c) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlals_s32(a, b, c);
#else
return HEDLEY_STATIC_CAST(int64_t, b) * HEDLEY_STATIC_CAST(int64_t, c) * 2 + a;
return simde_vqaddd_s64(a, simde_vqdmulls_s32(b, c));
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......@@ -72,8 +69,7 @@ simde_vqdmlal_s16(simde_int32x4_t a, simde_int16x4_t b, simde_int16x4_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlal_s16(a, b, c);
#else
simde_int32x4_t temp = simde_vmulq_s32(simde_vmovl_s16(b), simde_vmovl_s16(c));
return simde_vqaddq_s32(simde_vqaddq_s32(temp, temp), a);
return simde_vqaddq_s32(simde_vqdmull_s16(b, c), a);
#endif
}
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
......@@ -87,10 +83,7 @@ simde_vqdmlal_s32(simde_int64x2_t a, simde_int32x2_t b, simde_int32x2_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlal_s32(a, b, c);
#else
simde_int64x2_t r = simde_x_vmulq_s64(
simde_vmovl_s32(b),
simde_vmovl_s32(c));
return simde_vqaddq_s64(a, simde_vqaddq_s64(r, r));
return simde_vqaddq_s64(simde_vqdmull_s32(b, c), a);
#endif
}
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
......
arm/neon/qdmlal_high.h
View file @ 69c9cd5c
......@@ -27,10 +27,9 @@
#if !defined(SIMDE_ARM_NEON_QDMLAL_HIGH_H)
#define SIMDE_ARM_NEON_QDMLAL_HIGH_H
#include "movl_high.h"
#include "mla.h"
#include "mul_n.h"
#include "types.h"
#include "qadd.h"
#include "qdmull_high.h"
HEDLEY_DIAGNOSTIC_PUSH
SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
......@@ -42,10 +41,7 @@ simde_vqdmlal_high_s16(simde_int32x4_t a, simde_int16x8_t b, simde_int16x8_t c)
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlal_high_s16(a, b, c);
#else
return simde_vaddq_s32(
simde_vmulq_n_s32(
simde_vmulq_s32(
simde_vmovl_high_s16(b), simde_vmovl_high_s16(c)), 2), a);
return simde_vqaddq_s32(simde_vqdmull_high_s16(b, c), a);
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......@@ -59,17 +55,7 @@ simde_vqdmlal_high_s32(simde_int64x2_t a, simde_int32x4_t b, simde_int32x4_t c)
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlal_high_s32(a, b, c);
#else
simde_int64x2_private r_ = simde_int64x2_to_private(
simde_x_vmulq_s64(
simde_vmovl_high_s32(b),
simde_vmovl_high_s32(c)));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vaddq_s64(a, simde_int64x2_from_private(r_));
return simde_vqaddq_s64(simde_vqdmull_high_s32(b, c), a);
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......
arm/neon/qdmlal_high_lane.h
View file @ 69c9cd5c
......@@ -27,92 +27,49 @@
#if !defined(SIMDE_ARM_NEON_QDMLAL_HIGH_LANE_H)
#define SIMDE_ARM_NEON_QDMLAL_HIGH_LANE_H
#include "movl_high.h"
#include "add.h"
#include "mul.h"
#include "mul_n.h"
#include "dup_n.h"
#include "mla.h"
#include "dup_lane.h"
#include "get_high.h"
#include "types.h"
#include "qdmlal.h"
HEDLEY_DIAGNOSTIC_PUSH
SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
SIMDE_BEGIN_DECLS_
SIMDE_FUNCTION_ATTRIBUTES
simde_int32x4_t
simde_vqdmlal_high_lane_s16(simde_int32x4_t a, simde_int16x8_t b, simde_int16x4_t v, const int lane) SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
return simde_vaddq_s32(
simde_vmulq_n_s32(
simde_vmulq_s32(
simde_vmovl_high_s16(b),
simde_vmovl_high_s16(simde_vdupq_n_s16(simde_int16x4_to_private(v).values[lane]))), 2), a);
}
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
#define simde_vqdmlal_high_lane_s16(a, b, v, lane) vqdmlal_high_lane_s16(a, b, v, lane)
#else
#define simde_vqdmlal_high_lane_s16(a, b, v, lane) simde_vqdmlal_s16((a), simde_vget_high_s16((b)), simde_vdup_lane_s16((v), (lane)))
#endif
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
#undef vqdmlal_high_lane_s16
#define vqdmlal_high_lane_s16(a, b, v, lane) simde_vqdmlal_high_lane_s16((a), (b), (v), (lane))
#define vqdmlal_high_lane_s16(a, b, c, lane) simde_vqdmlal_high_lane_s16((a), (b), (c), (lane))
#endif
SIMDE_FUNCTION_ATTRIBUTES
simde_int32x4_t
simde_vqdmlal_high_laneq_s16(simde_int32x4_t a, simde_int16x8_t b, simde_int16x8_t v, const int lane) SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 7) {
return simde_vaddq_s32(
simde_vmulq_n_s32(
simde_vmulq_s32(
simde_vmovl_high_s16(b),
simde_vmovl_high_s16(simde_vdupq_n_s16(simde_int16x8_to_private(v).values[lane]))), 2), a);
}
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
#define simde_vqdmlal_high_laneq_s16(a, b, v, lane) vqdmlal_high_laneq_s16(a, b, v, lane)
#else
#define simde_vqdmlal_high_laneq_s16(a, b, v, lane) simde_vqdmlal_s16((a), simde_vget_high_s16((b)), simde_vdup_laneq_s16((v), (lane)))
#endif
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
#undef vqdmlal_high_laneq_s16
#define vqdmlal_high_laneq_s16(a, b, v, lane) simde_vqdmlal_high_laneq_s16((a), (b), (v), (lane))
#endif
SIMDE_FUNCTION_ATTRIBUTES
simde_int64x2_t
simde_vqdmlal_high_lane_s32(simde_int64x2_t a, simde_int32x4_t b, simde_int32x2_t v, const int lane) SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 1) {
simde_int64x2_private r_ = simde_int64x2_to_private(
simde_x_vmulq_s64(
simde_vmovl_high_s32(b),
simde_vmovl_high_s32(simde_vdupq_n_s32(simde_int32x2_to_private(v).values[lane]))));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vaddq_s64(a, simde_int64x2_from_private(r_));
}
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
#define simde_vqdmlal_high_lane_s32(a, b, v, lane) vqdmlal_high_lane_s32(a, b, v, lane)
#else
#define simde_vqdmlal_high_lane_s32(a, b, v, lane) simde_vqdmlal_s32((a), simde_vget_high_s32((b)), simde_vdup_lane_s32((v), (lane)))
#endif
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
#undef vqdmlal_high_lane_s32
#define vqdmlal_high_lane_s32(a, b, v, lane) simde_vqdmlal_high_lane_s32((a), (b), (v), (lane))
#endif
SIMDE_FUNCTION_ATTRIBUTES
simde_int64x2_t
simde_vqdmlal_high_laneq_s32(simde_int64x2_t a, simde_int32x4_t b, simde_int32x4_t v, const int lane) SIMDE_REQUIRE_CONSTANT_RANGE(lane, 0, 3) {
simde_int64x2_private r_ = simde_int64x2_to_private(
simde_x_vmulq_s64(
simde_vmovl_high_s32(b),
simde_vmovl_high_s32(simde_vdupq_n_s32(simde_int32x4_to_private(v).values[lane]))));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vaddq_s64(a, simde_int64x2_from_private(r_));
}
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
#define simde_vqdmlal_high_laneq_s32(a, b, v, lane) vqdmlal_high_laneq_s32(a, b, v, lane)
#else
#define simde_vqdmlal_high_laneq_s32(a, b, v, lane) simde_vqdmlal_s32((a), simde_vget_high_s32((b)), simde_vdup_laneq_s32((v), (lane)))
#endif
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
#undef vqdmlal_high_laneq_s32
......
arm/neon/qdmlal_high_n.h
View file @ 69c9cd5c
......@@ -27,12 +27,9 @@
#if !defined(SIMDE_ARM_NEON_QDMLAL_HIGH_N_H)
#define SIMDE_ARM_NEON_QDMLAL_HIGH_N_H
#include "movl_high.h"
#include "dup_n.h"
#include "add.h"
#include "mul.h"
#include "mul_n.h"
#include "types.h"
#include "qdmlal_high.h"
HEDLEY_DIAGNOSTIC_PUSH
SIMDE_DISABLE_UNWANTED_DIAGNOSTICS
......@@ -44,11 +41,7 @@ simde_vqdmlal_high_n_s16(simde_int32x4_t a, simde_int16x8_t b, int16_t c) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlal_high_n_s16(a, b, c);
#else
return simde_vaddq_s32(
simde_vmulq_n_s32(
simde_vmulq_s32(
simde_vmovl_high_s16(b),
simde_vmovl_high_s16(simde_vdupq_n_s16(c))), 2), a);
return simde_vqdmlal_high_s16(a, b, simde_vdupq_n_s16(c));
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......@@ -62,17 +55,7 @@ simde_vqdmlal_high_n_s32(simde_int64x2_t a, simde_int32x4_t b, int32_t c) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqdmlal_high_n_s32(a, b, c);
#else
simde_int64x2_private r_ = simde_int64x2_to_private(
simde_x_vmulq_s64(
simde_vmovl_high_s32(b),
simde_vmovl_high_s32(simde_vdupq_n_s32(c))));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vaddq_s64(a, simde_int64x2_from_private(r_));
return simde_vqdmlal_high_s32(a, b, simde_vdupq_n_s32(c));
#endif
}
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment