Skip to content

S
simde
Commit 737e3b33 authored by Zhijin Zeng's avatar Zhijin Zeng Committed by Michael R. Crusoe
Browse files
Options

arm: fix some neon2rvv intrinsic function error 

1. For vqdmlal_s16/s32: the doubling result maybe overflow,
so need to use vqaddq_s16/32 to saturate it. As the same with
vqdmlsl_s16/32.

2. The vrdmulh family function need to use vqadd saturating
function to avoid the doubling result overflow.

3. The result of vrshl family function need to keep the sign
bit of the origin data. If a > 0 && b < 0, the result of
(a + (1 << (-b - 1))) maybe overflow into a negative value.
And in gcc/clang, >> means the arithmetic shift left, so it
will get the incorrect sign bit whithout unsigned extend value.
Signed-off-by: default avatarZhijin Zeng <zhijin.zeng@spacemit.com>
parent 5242a77d
Show whitespace changes
Inline Side-by-side
Showing with 59 additions and 36 deletions
arm/neon/qdmlal.h
View file @ 737e3b33
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include "mul.h" #include "mul.h"
#include "mul_n.h" #include "mul_n.h"
#include "movl.h" #include "movl.h"
#include "qadd.h"
#include "types.h" #include "types.h"
HEDLEY_DIAGNOSTIC_PUSH HEDLEY_DIAGNOSTIC_PUSH
...@@ -71,7 +72,8 @@ simde_vqdmlal_s16(simde_int32x4_t a, simde_int16x4_t b, simde_int16x4_t c) { ...@@ -71,7 +72,8 @@ simde_vqdmlal_s16(simde_int32x4_t a, simde_int16x4_t b, simde_int16x4_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) #if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlal_s16(a, b, c); return vqdmlal_s16(a, b, c);
#else #else
return simde_vaddq_s32(simde_vmulq_n_s32(simde_vmulq_s32(simde_vmovl_s16(b), simde_vmovl_s16(c)), 2), a); simde_int32x4_t temp = simde_vmulq_s32(simde_vmovl_s16(b), simde_vmovl_s16(c));
return simde_vqaddq_s32(simde_vqaddq_s32(temp, temp), a);
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
...@@ -85,17 +87,10 @@ simde_vqdmlal_s32(simde_int64x2_t a, simde_int32x2_t b, simde_int32x2_t c) { ...@@ -85,17 +87,10 @@ simde_vqdmlal_s32(simde_int64x2_t a, simde_int32x2_t b, simde_int32x2_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) #if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlal_s32(a, b, c); return vqdmlal_s32(a, b, c);
#else #else
simde_int64x2_private r_ = simde_int64x2_to_private( simde_int64x2_t r = simde_x_vmulq_s64(
simde_x_vmulq_s64(
simde_vmovl_s32(b), simde_vmovl_s32(b),
simde_vmovl_s32(c))); simde_vmovl_s32(c));
return simde_vqaddq_s64(a, simde_vqaddq_s64(r, r));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vaddq_s64(a, simde_int64x2_from_private(r_));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
......
arm/neon/qdmlsl.h
View file @ 737e3b33
...@@ -31,6 +31,8 @@ ...@@ -31,6 +31,8 @@
#include "mul.h" #include "mul.h"
#include "mul_n.h" #include "mul_n.h"
#include "movl.h" #include "movl.h"
#include "qadd.h"
#include "qsub.h"
#include "types.h" #include "types.h"
HEDLEY_DIAGNOSTIC_PUSH HEDLEY_DIAGNOSTIC_PUSH
...@@ -71,7 +73,8 @@ simde_vqdmlsl_s16(simde_int32x4_t a, simde_int16x4_t b, simde_int16x4_t c) { ...@@ -71,7 +73,8 @@ simde_vqdmlsl_s16(simde_int32x4_t a, simde_int16x4_t b, simde_int16x4_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) #if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlsl_s16(a, b, c); return vqdmlsl_s16(a, b, c);
#else #else
return simde_vsubq_s32(a, simde_vmulq_n_s32(simde_vmulq_s32(simde_vmovl_s16(b), simde_vmovl_s16(c)), 2)); simde_int32x4_t temp = simde_vmulq_s32(simde_vmovl_s16(b), simde_vmovl_s16(c));
return simde_vqsubq_s32(a, simde_vqaddq_s32(temp, temp));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
...@@ -85,17 +88,10 @@ simde_vqdmlsl_s32(simde_int64x2_t a, simde_int32x2_t b, simde_int32x2_t c) { ...@@ -85,17 +88,10 @@ simde_vqdmlsl_s32(simde_int64x2_t a, simde_int32x2_t b, simde_int32x2_t c) {
#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) #if defined(SIMDE_ARM_NEON_A32V7_NATIVE)
return vqdmlsl_s32(a, b, c); return vqdmlsl_s32(a, b, c);
#else #else
simde_int64x2_private r_ = simde_int64x2_to_private( simde_int64x2_t r = simde_x_vmulq_s64(
simde_x_vmulq_s64(
simde_vmovl_s32(b), simde_vmovl_s32(b),
simde_vmovl_s32(c))); simde_vmovl_s32(c));
return simde_vqsubq_s64(a, simde_vqaddq_s64(r, r));
SIMDE_VECTORIZE
for (size_t i = 0 ; i < (sizeof(r_.values) / sizeof(r_.values[0])) ; i++) {
r_.values[i] = r_.values[i] * HEDLEY_STATIC_CAST(int64_t, 2);
}
return simde_vsubq_s64(a, simde_int64x2_from_private(r_));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A32V7_ENABLE_NATIVE_ALIASES)
......
arm/neon/qrdmulh.h
View file @ 737e3b33
...@@ -40,7 +40,10 @@ simde_vqrdmulhh_s16(int16_t a, int16_t b) { ...@@ -40,7 +40,10 @@ simde_vqrdmulhh_s16(int16_t a, int16_t b) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) #if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqrdmulhh_s16(a, b); return vqrdmulhh_s16(a, b);
#else #else
return HEDLEY_STATIC_CAST(int16_t, (((1 << 15) + ((HEDLEY_STATIC_CAST(int32_t, (HEDLEY_STATIC_CAST(int32_t, a) * HEDLEY_STATIC_CAST(int32_t, b)))) << 1)) >> 16) & 0xffff); int32_t temp = HEDLEY_STATIC_CAST(int32_t, a) * HEDLEY_STATIC_CAST(int32_t, b);
int32_t r = temp > 0 ? (temp > (INT32_MAX >> 1) ? INT32_MAX : (temp << 1)) : (temp < (INT32_MIN >> 1) ? INT32_MIN : (temp << 1));
r = (r > (INT32_MAX - (1 << 15))) ? INT32_MAX : ((1 << 15) + r);
return HEDLEY_STATIC_CAST(int16_t, ((r >> 16) & 0xffff));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
...@@ -54,7 +57,10 @@ simde_vqrdmulhs_s32(int32_t a, int32_t b) { ...@@ -54,7 +57,10 @@ simde_vqrdmulhs_s32(int32_t a, int32_t b) {
#if defined(SIMDE_ARM_NEON_A64V8_NATIVE) #if defined(SIMDE_ARM_NEON_A64V8_NATIVE)
return vqrdmulhs_s32(a, b); return vqrdmulhs_s32(a, b);
#else #else
return HEDLEY_STATIC_CAST(int32_t, (((HEDLEY_STATIC_CAST(int64_t, 1) << 31) + ((HEDLEY_STATIC_CAST(int64_t, (HEDLEY_STATIC_CAST(int64_t, a) * HEDLEY_STATIC_CAST(int64_t, b)))) << 1)) >> 32) & 0xffffffff); int64_t temp = HEDLEY_STATIC_CAST(int64_t, a) * HEDLEY_STATIC_CAST(int64_t, b);
int64_t r = temp > 0 ? (temp > (INT64_MAX >> 1) ? INT64_MAX : (temp << 1)) : (temp < (INT64_MIN >> 1) ? INT64_MIN : (temp << 1));
r = (r > (INT64_MAX - (HEDLEY_STATIC_CAST(int64_t, 1) << 31))) ? INT64_MAX : ((HEDLEY_STATIC_CAST(int64_t, 1) << 31) + r);
return HEDLEY_STATIC_CAST(int32_t, ((r >> 32) & 0xffffffff));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
......
arm/neon/qrshl.h
View file @ 737e3b33
...@@ -44,7 +44,10 @@ simde_vqrshlb_s8(int8_t a, int8_t b) { ...@@ -44,7 +44,10 @@ simde_vqrshlb_s8(int8_t a, int8_t b) {
if (b < -8) { if (b < -8) {
r = 0; r = 0;
} else if (b < 0) { } else if (b < 0) {
r = HEDLEY_STATIC_CAST(int8_t, ((a + (1 << (-b - 1))) >> -b)); r = HEDLEY_STATIC_CAST(int8_t, a <= 0
? ((a + (1 << (-b - 1))) >> -b)
: HEDLEY_STATIC_CAST(int8_t, ((HEDLEY_STATIC_CAST(uint8_t,
(a + (1 << (-b - 1)))) >> -b) & 0x7FUL)));
} else if (b == 0) { } else if (b == 0) {
r = a; r = a;
} else if (b < 7) { } else if (b < 7) {
...@@ -79,7 +82,10 @@ simde_vqrshlh_s16(int16_t a, int16_t b) { ...@@ -79,7 +82,10 @@ simde_vqrshlh_s16(int16_t a, int16_t b) {
if (b8 <= -16) { if (b8 <= -16) {
r = 0; r = 0;
} else if (b8 < 0) { } else if (b8 < 0) {
r = HEDLEY_STATIC_CAST(int16_t, ((a + (1 << (-b8 - 1))) >> -b8)); r = HEDLEY_STATIC_CAST(int16_t, a <= 0
? ((a + (1 << (-b8 - 1))) >> -b8)
: HEDLEY_STATIC_CAST(int16_t, ((HEDLEY_STATIC_CAST(uint16_t,
(a + (1 << (-b8 - 1)))) >> -b8) & 0x7FFFUL)));
} else if (b8 == 0) { } else if (b8 == 0) {
r = a; r = a;
} else if (b8 < 15) { } else if (b8 < 15) {
...@@ -114,7 +120,10 @@ simde_vqrshls_s32(int32_t a, int32_t b) { ...@@ -114,7 +120,10 @@ simde_vqrshls_s32(int32_t a, int32_t b) {
if (b8 <= -32) { if (b8 <= -32) {
r = 0; r = 0;
} else if (b8 < 0) { } else if (b8 < 0) {
r = ((a + (1 << (-b8 - 1))) >> -b8); r = a <= 0
? ((a + (1 << (-b8 - 1))) >> -b8)
: HEDLEY_STATIC_CAST(int32_t, ((HEDLEY_STATIC_CAST(uint32_t,
(a + (1 << (-b8 - 1)))) >> -b8) & 0x7FFFFFFFUL));
} else if (b8 == 0) { } else if (b8 == 0) {
r = a; r = a;
} else if (b8 < 31) { } else if (b8 < 31) {
...@@ -149,7 +158,10 @@ simde_vqrshld_s64(int64_t a, int64_t b) { ...@@ -149,7 +158,10 @@ simde_vqrshld_s64(int64_t a, int64_t b) {
if (b8 <= -64) { if (b8 <= -64) {
r = 0; r = 0;
} else if (b8 < 0) { } else if (b8 < 0) {
r = ((a + (INT64_C(1) << (-b8 - 1))) >> -b8); r = a <= 0
? ((a + (INT64_C(1) << (-b8 - 1))) >> -b8)
: HEDLEY_STATIC_CAST(int64_t, ((HEDLEY_STATIC_CAST(uint64_t,
(a + (INT64_C(1) << (-b8 - 1)))) >> -b8) & 0x7FFFFFFFFFFFFFFFUL));
} else if (b8 == 0) { } else if (b8 == 0) {
r = a; r = a;
} else if (b8 < 63) { } else if (b8 < 63) {
......
arm/neon/rshl.h
View file @ 737e3b33
...@@ -84,7 +84,9 @@ simde_vrshld_s64(int64_t a, int64_t b) { ...@@ -84,7 +84,9 @@ simde_vrshld_s64(int64_t a, int64_t b) {
? 0 ? 0
: (b >= 0) : (b >= 0)
? (a << b) ? (a << b)
: ((a + (INT64_C(1) << (-b - 1))) >> -b); : (a <= 0
? ((a + (INT64_C(1) << (-b - 1))) >> -b)
: HEDLEY_STATIC_CAST(int64_t, (HEDLEY_STATIC_CAST(uint64_t, (a + (INT64_C(1) << (-b - 1)))) >> -b)));
#endif #endif
} }
#if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES) #if defined(SIMDE_ARM_NEON_A64V8_ENABLE_NATIVE_ALIASES)
...@@ -148,7 +150,9 @@ simde_vrshl_s8 (const simde_int8x8_t a, const simde_int8x8_t b) { ...@@ -148,7 +150,9 @@ simde_vrshl_s8 (const simde_int8x8_t a, const simde_int8x8_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int8_t, r_.values[i] = HEDLEY_STATIC_CAST(int8_t,
(simde_math_abs(b_.values[i]) >= 8) ? 0 : (simde_math_abs(b_.values[i]) >= 8) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int8_t, ((HEDLEY_STATIC_CAST(uint8_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0x7FUL)))));
} }
#endif #endif
...@@ -189,7 +193,9 @@ simde_vrshl_s16 (const simde_int16x4_t a, const simde_int16x4_t b) { ...@@ -189,7 +193,9 @@ simde_vrshl_s16 (const simde_int16x4_t a, const simde_int16x4_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int16_t, r_.values[i] = HEDLEY_STATIC_CAST(int16_t,
(simde_math_abs(b_.values[i]) >= 16) ? 0 : (simde_math_abs(b_.values[i]) >= 16) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int16_t, ((HEDLEY_STATIC_CAST(uint16_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0x7FFFUL)))));
} }
#endif #endif
...@@ -230,7 +236,9 @@ simde_vrshl_s32 (const simde_int32x2_t a, const simde_int32x2_t b) { ...@@ -230,7 +236,9 @@ simde_vrshl_s32 (const simde_int32x2_t a, const simde_int32x2_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int32_t, r_.values[i] = HEDLEY_STATIC_CAST(int32_t,
(simde_math_abs(b_.values[i]) >= 32) ? 0 : (simde_math_abs(b_.values[i]) >= 32) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int32_t, ((HEDLEY_STATIC_CAST(uint32_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0x7FFFFFFFUL)))));
} }
#endif #endif
...@@ -513,7 +521,9 @@ simde_vrshlq_s8 (const simde_int8x16_t a, const simde_int8x16_t b) { ...@@ -513,7 +521,9 @@ simde_vrshlq_s8 (const simde_int8x16_t a, const simde_int8x16_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int8_t, r_.values[i] = HEDLEY_STATIC_CAST(int8_t,
(simde_math_abs(b_.values[i]) >= 8) ? 0 : (simde_math_abs(b_.values[i]) >= 8) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int8_t, ((HEDLEY_STATIC_CAST(uint8_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0x7FUL)))));
} }
#endif #endif
...@@ -580,7 +590,9 @@ simde_vrshlq_s16 (const simde_int16x8_t a, const simde_int16x8_t b) { ...@@ -580,7 +590,9 @@ simde_vrshlq_s16 (const simde_int16x8_t a, const simde_int16x8_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int16_t, r_.values[i] = HEDLEY_STATIC_CAST(int16_t,
(simde_math_abs(b_.values[i]) >= 16) ? 0 : (simde_math_abs(b_.values[i]) >= 16) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int16_t, ((HEDLEY_STATIC_CAST(uint16_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0x7FFFUL)))));
} }
#endif #endif
...@@ -635,7 +647,9 @@ simde_vrshlq_s32 (const simde_int32x4_t a, const simde_int32x4_t b) { ...@@ -635,7 +647,9 @@ simde_vrshlq_s32 (const simde_int32x4_t a, const simde_int32x4_t b) {
r_.values[i] = HEDLEY_STATIC_CAST(int32_t, r_.values[i] = HEDLEY_STATIC_CAST(int32_t,
(simde_math_abs(b_.values[i]) >= 32) ? 0 : (simde_math_abs(b_.values[i]) >= 32) ? 0 :
(b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) : (b_.values[i] >= 0) ? (a_.values[i] << b_.values[i]) :
((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i])); ((a_.values[i] <= 0) ? ((a_.values[i] + (1 << (-b_.values[i] - 1))) >> -b_.values[i]) :
HEDLEY_STATIC_CAST(int32_t, ((HEDLEY_STATIC_CAST(uint32_t,
(a_.values[i] + (1 << (-b_.values[i] - 1)))) >> -b_.values[i]) & (0X7FFFFFFFUL)))));
} }
#endif #endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment