Commit 889b4b00 authored by Laurent THOMAS's avatar Laurent THOMAS

SIMD optimizations for Neon/AVX2/AVX512 in the PDSCH transmit path

parent ff58b5e1
...@@ -24,8 +24,10 @@ ...@@ -24,8 +24,10 @@
#include "executables/softmodem-common.h" #include "executables/softmodem-common.h"
// #define DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // TODO: For debug, to be removed if want to merge to develop // #define DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // TODO: For debug, to be removed if want to merge to develop
// #define DEBUG_LAYER_MAPPING
//Table 6.3.1.5-1 Precoding Matrix W 1 layer 2 antenna ports 'n' = -1 and 'o' = -j #define USE_NEON
// #define USE_GATHER
// Table 6.3.1.5-1 Precoding Matrix W 1 layer 2 antenna ports 'n' = -1 and 'o' = -j
const char nr_W_1l_2p[6][2][1] = { const char nr_W_1l_2p[6][2][1] = {
{{'1'}, {'0'}}, // pmi 0 {{'1'}, {'0'}}, // pmi 0
{{'0'}, {'1'}}, {{'0'}, {'1'}},
...@@ -35,7 +37,7 @@ const char nr_W_1l_2p[6][2][1] = { ...@@ -35,7 +37,7 @@ const char nr_W_1l_2p[6][2][1] = {
{{'1'}, {'o'}} // pmi 5 {{'1'}, {'o'}} // pmi 5
}; };
//Table 6.3.1.5-3 Precoding Matrix W 1 layer 4 antenna ports 'n' = -1 and 'o' = -j // Table 6.3.1.5-3 Precoding Matrix W 1 layer 4 antenna ports 'n' = -1 and 'o' = -j
const char nr_W_1l_4p[28][4][1] = { const char nr_W_1l_4p[28][4][1] = {
{{'1'}, {'0'}, {'0'}, {'0'}}, // pmi 0 {{'1'}, {'0'}, {'0'}, {'0'}}, // pmi 0
{{'0'}, {'1'}, {'0'}, {'0'}}, {{'0'}, {'1'}, {'0'}, {'0'}},
...@@ -68,14 +70,14 @@ const char nr_W_1l_4p[28][4][1] = { ...@@ -68,14 +70,14 @@ const char nr_W_1l_4p[28][4][1] = {
{{'1'}, {'o'}, {'o'}, {'n'}} // pmi 27 {{'1'}, {'o'}, {'o'}, {'n'}} // pmi 27
}; };
//Table 6.3.1.5-4 Precoding Matrix W 2 antenna ports layers 2 'n' = -1 and 'o' = -j // Table 6.3.1.5-4 Precoding Matrix W 2 antenna ports layers 2 'n' = -1 and 'o' = -j
const char nr_W_2l_2p[3][2][2] = { const char nr_W_2l_2p[3][2][2] = {
{{'1', '0'}, {'0', '1'}}, // pmi 0 {{'1', '0'}, {'0', '1'}}, // pmi 0
{{'1', '1'}, {'1', 'n'}}, {{'1', '1'}, {'1', 'n'}},
{{'1', '1'}, {'j', 'o'}} // pmi 2 {{'1', '1'}, {'j', 'o'}} // pmi 2
}; };
//Table 6.3.1.5-5 Precoding Matrix W 2 layers 4 antenna ports 'n' = -1 and 'o' = -j // Table 6.3.1.5-5 Precoding Matrix W 2 layers 4 antenna ports 'n' = -1 and 'o' = -j
const char nr_W_2l_4p[22][4][2] = { const char nr_W_2l_4p[22][4][2] = {
{{'1', '0'}, {'0', '1'}, {'0', '0'}, {'0', '0'}}, // pmi 0 {{'1', '0'}, {'0', '1'}, {'0', '0'}, {'0', '0'}}, // pmi 0
{{'1', '0'}, {'0', '0'}, {'0', '1'}, {'0', '0'}}, {{'1', '0'}, {'0', '0'}, {'0', '0'}, {'0', '1'}}, {{'1', '0'}, {'0', '0'}, {'0', '1'}, {'0', '0'}}, {{'1', '0'}, {'0', '0'}, {'0', '0'}, {'0', '1'}},
...@@ -94,7 +96,7 @@ const char nr_W_2l_4p[22][4][2] = { ...@@ -94,7 +96,7 @@ const char nr_W_2l_4p[22][4][2] = {
{{'1', '1'}, {'o', 'o'}, {'1', 'n'}, {'o', 'j'}}, {{'1', '1'}, {'o', 'o'}, {'j', 'o'}, {'1', 'n'}} // pmi 21 {{'1', '1'}, {'o', 'o'}, {'1', 'n'}, {'o', 'j'}}, {{'1', '1'}, {'o', 'o'}, {'j', 'o'}, {'1', 'n'}} // pmi 21
}; };
//Table 6.3.1.5-6 Precoding Matrix W 3 layers 4 antenna ports 'n' = -1 and 'o' = -j // Table 6.3.1.5-6 Precoding Matrix W 3 layers 4 antenna ports 'n' = -1 and 'o' = -j
const char nr_W_3l_4p[7][4][3] = {{{'1', '0', '0'}, {'0', '1', '0'}, {'0', '0', '1'}, {'0', '0', '0'}}, // pmi 0 const char nr_W_3l_4p[7][4][3] = {{{'1', '0', '0'}, {'0', '1', '0'}, {'0', '0', '1'}, {'0', '0', '0'}}, // pmi 0
{{'1', '0', '0'}, {'0', '1', '0'}, {'1', '0', '0'}, {'0', '0', '1'}}, {{'1', '0', '0'}, {'0', '1', '0'}, {'1', '0', '0'}, {'0', '0', '1'}},
{{'1', '0', '0'}, {'0', '1', '0'}, {'n', '0', '0'}, {'0', '0', '1'}}, {{'1', '0', '0'}, {'0', '1', '0'}, {'n', '0', '0'}, {'0', '0', '1'}},
...@@ -103,7 +105,7 @@ const char nr_W_3l_4p[7][4][3] = {{{'1', '0', '0'}, {'0', '1', '0'}, {'0', '0', ...@@ -103,7 +105,7 @@ const char nr_W_3l_4p[7][4][3] = {{{'1', '0', '0'}, {'0', '1', '0'}, {'0', '0',
{{'1', '1', '1'}, {'n', '1', 'n'}, {'1', '1', 'n'}, {'n', '1', '1'}}, {{'1', '1', '1'}, {'n', '1', 'n'}, {'1', '1', 'n'}, {'n', '1', '1'}},
{{'1', '1', '1'}, {'n', '1', 'n'}, {'j', 'j', 'o'}, {'o', 'j', 'j'}}}; {{'1', '1', '1'}, {'n', '1', 'n'}, {'j', 'j', 'o'}, {'o', 'j', 'j'}}};
//Table 6.3.1.5-7 Precoding Matrix W 4 layers 4 antenna ports 'n' = -1 and 'o' = -j // Table 6.3.1.5-7 Precoding Matrix W 4 layers 4 antenna ports 'n' = -1 and 'o' = -j
const char nr_W_4l_4p[5][4][4] = { const char nr_W_4l_4p[5][4][4] = {
{{'1', '0', '0', '0'}, {'0', '1', '0', '0'}, {'0', '0', '1', '0'}, {'0', '0', '0', '1'}}, // pmi 0 {{'1', '0', '0', '0'}, {'0', '1', '0', '0'}, {'0', '0', '1', '0'}, {'0', '0', '0', '1'}}, // pmi 0
{{'1', '1', '0', '0'}, {'0', '0', '1', '1'}, {'1', 'n', '0', '0'}, {'0', '0', '1', 'n'}}, {{'1', '1', '0', '0'}, {'0', '0', '1', '1'}, {'1', 'n', '0', '0'}, {'0', '0', '1', 'n'}},
...@@ -112,137 +114,133 @@ const char nr_W_4l_4p[5][4][4] = { ...@@ -112,137 +114,133 @@ const char nr_W_4l_4p[5][4][4] = {
{{'1', '1', '1', '1'}, {'1', 'n', '1', 'n'}, {'j', 'j', 'o', 'o'}, {'j', 'o', 'o', 'j'}} // pmi 4 {{'1', '1', '1', '1'}, {'1', 'n', '1', 'n'}, {'j', 'j', 'o', 'o'}, {'j', 'o', 'o', 'j'}} // pmi 4
}; };
void nr_modulation(const uint32_t *in, void nr_modulation(const uint32_t *in, uint32_t length, uint16_t mod_order, int16_t *out)
uint32_t length,
uint16_t mod_order,
int16_t *out)
{ {
uint16_t mask = ((1<<mod_order)-1); uint16_t mask = ((1 << mod_order) - 1);
int32_t* nr_mod_table32; int32_t *nr_mod_table32;
int32_t* out32 = (int32_t*) out; int32_t *out32 = (int32_t *)out;
const uint8_t *in_bytes = (const uint8_t *)in; const uint8_t *in_bytes = (const uint8_t *)in;
const uint64_t *in64 = (const uint64_t *)in; const uint64_t *in64 = (const uint64_t *)in;
int64_t* out64 = (int64_t*) out; int64_t *out64 = (int64_t *)out;
uint32_t i=0; uint32_t i = 0;
#if defined(__SSE2__) #if defined(__SSE2__)
simde__m128i *nr_mod_table128; simde__m128i *nr_mod_table128;
simde__m128i *out128; simde__m128i *out128;
#endif #endif
LOG_D(PHY,"nr_modulation: length %d, mod_order %d\n",length,mod_order); LOG_D(PHY, "nr_modulation: length %d, mod_order %d\n", length, mod_order);
switch (mod_order) { switch (mod_order) {
#if defined(__SSE2__) #if defined(__SSE2__)
case 2: case 2:
nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table; nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table;
out128 = (simde__m128i *)out; out128 = (simde__m128i *)out;
for (i=0; i<length/8; i++) for (i = 0; i < length / 8; i++)
out128[i] = nr_mod_table128[in_bytes[i]]; out128[i] = nr_mod_table128[in_bytes[i]];
// the bits that are left out // the bits that are left out
i = i*8/2; i = i * 8 / 2;
nr_mod_table32 = (int32_t*) nr_qpsk_mod_table; nr_mod_table32 = (int32_t *)nr_qpsk_mod_table;
while (i<length/2) { while (i < length / 2) {
const int idx = ((in_bytes[(i * 2) / 8] >> ((i * 2) & 0x7)) & mask); const int idx = ((in_bytes[(i * 2) / 8] >> ((i * 2) & 0x7)) & mask);
out32[i] = nr_mod_table32[idx]; out32[i] = nr_mod_table32[idx];
i++; i++;
} }
return; return;
#else #else
case 2: case 2:
nr_mod_table32 = (int32_t*) nr_qpsk_mod_table; nr_mod_table32 = (int32_t *)nr_qpsk_mod_table;
for (i=0; i<length/mod_order; i++) { for (i = 0; i < length / mod_order; i++) {
const int idx = ((in[i * 2 / 32] >> ((i * 2) & 0x1f)) & mask); const int idx = ((in[i * 2 / 32] >> ((i * 2) & 0x1f)) & mask);
out32[i] = nr_mod_table32[idx]; out32[i] = nr_mod_table32[idx];
} }
return; return;
#endif #endif
case 4: case 4:
out64 = (int64_t*) out; out64 = (int64_t *)out;
for (i=0; i<length/8; i++) for (i = 0; i < length / 8; i++)
out64[i] = nr_16qam_byte_mod_table[in_bytes[i]]; out64[i] = nr_16qam_byte_mod_table[in_bytes[i]];
// the bits that are left out // the bits that are left out
i = i*8/4; i = i * 8 / 4;
while (i<length/4) { while (i < length / 4) {
const int idx = ((in_bytes[(i * 4) / 8] >> ((i * 4) & 0x7)) & mask); const int idx = ((in_bytes[(i * 4) / 8] >> ((i * 4) & 0x7)) & mask);
out32[i] = nr_16qam_mod_table[idx]; out32[i] = nr_16qam_mod_table[idx];
i++; i++;
} }
return; return;
case 6: case 6:
if (length > (3*64)) if (length > (3 * 64))
for (i = 0; i < length - 3 * 64; i += 3 * 64) { for (i = 0; i < length - 3 * 64; i += 3 * 64) {
uint64_t x = *in64++; uint64_t x = *in64++;
uint64_t x1 = x & 0xfff; uint64_t x1 = x & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x >> 12) & 0xfff; x1 = (x >> 12) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x >> 24) & 0xfff; x1 = (x >> 24) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x >> 36) & 0xfff; x1 = (x >> 36) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x >> 48) & 0xfff; x1 = (x >> 48) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
uint64_t x2 = (x >> 60); uint64_t x2 = (x >> 60);
x = *in64++; x = *in64++;
x2 |= x<<4; x2 |= x << 4;
x1 = x2 & 0xfff; x1 = x2 & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 12) & 0xfff; x1 = (x2 >> 12) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 24) & 0xfff; x1 = (x2 >> 24) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 36) & 0xfff; x1 = (x2 >> 36) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 48) & 0xfff; x1 = (x2 >> 48) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x2 = ((x>>56)&0xf0) | (x2>>60); x2 = ((x >> 56) & 0xf0) | (x2 >> 60);
x = *in64++; x = *in64++;
x2 |= x<<8; x2 |= x << 8;
x1 = x2 & 0xfff; x1 = x2 & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 12) & 0xfff; x1 = (x2 >> 12) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 24) & 0xfff; x1 = (x2 >> 24) & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 36) & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 48) & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
x2 = ((x >> 52) & 0xff0) | (x2 >> 60);
*out64++ = nr_64qam_mod_table[x2];
}
while (i + 24 <= length) {
uint32_t xx = 0;
memcpy(&xx, in_bytes + i / 8, 3);
uint64_t x1 = xx & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 36) & 0xfff; x1 = (xx >> 12) & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x1 = (x2 >> 48) & 0xfff; i += 24;
}
if (i != length) {
uint32_t xx = 0;
memcpy(&xx, in_bytes + i / 8, 2);
uint64_t x1 = xx & 0xfff;
*out64++ = nr_64qam_mod_table[x1]; *out64++ = nr_64qam_mod_table[x1];
x2 = ((x>>52)&0xff0) | (x2>>60);
*out64++ = nr_64qam_mod_table[x2];
} }
return;
while (i + 24 <= length) {
uint32_t xx = 0;
memcpy(&xx, in_bytes + i / 8, 3);
uint64_t x1 = xx & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
x1 = (xx >> 12) & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
i += 24;
}
if (i != length) {
uint32_t xx = 0;
memcpy(&xx, in_bytes + i / 8, 2);
uint64_t x1 = xx & 0xfff;
*out64++ = nr_64qam_mod_table[x1];
}
return;
case 8: case 8:
nr_mod_table32 = (int32_t*) nr_256qam_mod_table; nr_mod_table32 = (int32_t *)nr_256qam_mod_table;
for (i=0; i<length/8; i++) for (i = 0; i < length / 8; i++)
out32[i] = nr_mod_table32[in_bytes[i]]; out32[i] = nr_mod_table32[in_bytes[i]];
return; return;
default: default:
break; break;
} }
AssertFatal(false,"Invalid or unsupported modulation order %d\n",mod_order); AssertFatal(false, "Invalid or unsupported modulation order %d\n", mod_order);
} }
void nr_layer_mapping(int nbCodes, void nr_layer_mapping(int nbCodes,
...@@ -251,36 +249,296 @@ void nr_layer_mapping(int nbCodes, ...@@ -251,36 +249,296 @@ void nr_layer_mapping(int nbCodes,
uint8_t n_layers, uint8_t n_layers,
int layerSz, int layerSz,
uint32_t n_symbs, uint32_t n_symbs,
c16_t tx_layer[layerSz], c16_t tx_layers[][layerSz])
int layer)
{ {
LOG_D(PHY,"Doing layer mapping for %d layers, %d symbols\n",n_layers,n_symbs); LOG_D(PHY, "Doing layer mapping for %d layers, %d symbols\n", n_layers, n_symbs);
switch (n_layers) { switch (n_layers) {
case 1: case 1:
memcpy(tx_layer, mod_symbs[0], n_symbs * sizeof(**mod_symbs)); memcpy(tx_layers[0], mod_symbs[0], n_symbs * sizeof(**mod_symbs));
break; break;
case 2: case 2: {
case 3: #if defined(__AVX512BW__)
case 4: int i = 0;
for (int i = 0; i < n_symbs / n_layers; i++) { __m512i perm2a = _mm512_set_epi32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
const c16_t *base = mod_symbs[0] + n_layers * i; __m512i perm2b = _mm512_set_epi32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1);
tx_layer[i] = base[layer]; #ifndef USE_GATHER
} __m512i a, b;
break; int j;
for (i = 0, j = 0; i < n_symbs >> 4; i += 2, j++) {
a = ((__m512i *)mod_symbs[0])[i];
b = ((__m512i *)mod_symbs[0])[i + 1];
((__m512i *)tx_layers[0])[j] = _mm512_permutex2var_epi32(a, perm2a, b);
((__m512i *)tx_layers[1])[j] = _mm512_permutex2var_epi32(a, perm2b, b);
}
if (i << 4 != n_symbs) {
for (int i2 = ((n_symbs >> 4) << 4); i2 < n_symbs; i2 += 2) {
tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
}
}
#else
for (i = 0; i < n_symbs; i += 32) {
*(__m512i *)(tx_layers[0] + i / 2) = _mm512_i32gather_epi32(perm2a, mod_symbs[0] + i, 4);
*(__m512i *)(tx_layers[1] + i / 2) = _mm512_i32gather_epi32(perm2b, mod_symbs[0] + i, 4);
}
for (; i < n_symbs; i += 2) {
tx_layers[0][i >> 1] = mod_symbs[0][i];
tx_layers[1][i >> 1] = mod_symbs[0][i + 1];
}
#endif
#elif defined(__aarch64__)
int i;
#ifdef USE_NEON
// SIMDe doesn't handle this properly, gcc up to 14.2 neither
uint8_t const perm0[16] = {0, 1, 2, 3, 8, 9, 10, 11, 4, 5, 6, 7, 12, 13, 14, 15};
uint8x16_t perm = vld1q_u8(perm0);
uint8x16_t d;
for (i = 0; i < n_symbs >> 2; i++) {
d = vqtbl1q_u8(((uint8x16_t *)mod_symbs[0])[i], perm);
((int64_t *)tx_layers[0])[i] = vgetq_lane_u64((uint64x2_t)d, 0);
((int64_t *)tx_layers[1])[i] = vgetq_lane_u64((uint64x2_t)d, 1);
}
if (i << 2 != n_symbs) {
for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 2) {
tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
}
}
#else
for (i = 0; i < n_symbs; i += 2) {
tx_layers[0][i >> 1] = mod_symbs[0][i];
tx_layers[1][i >> 1] = mod_symbs[0][i + 1];
}
#endif
#else
int i;
simde__m256i perm2 = simde_mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);
simde__m256i d;
for (i = 0; i < n_symbs >> 3; i++) {
d = simde_mm256_permutevar8x32_epi32(((simde__m256i *)mod_symbs[0])[i], perm2);
((simde__m128i *)tx_layers[0])[i] = simde_mm256_extractf128_si256(d, 0);
((simde__m128i *)tx_layers[1])[i] = simde_mm256_extractf128_si256(d, 1);
}
if (i << 3 != n_symbs) {
for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 2) {
tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
}
}
#endif
} break;
case 3: {
#if defined(__AVX512BW__)
__m512i i0, i1, i2, d0;
__m512i perm3_0 =
_mm512_set_epi32(13 + 16, 10 + 16, 7 + 16, 4 + 16, 1 + 16, 14 + 16, 11 + 16, 8 + 16, 5 + 16, 2 + 16, 15, 12, 9, 6, 3, 0);
__m512i perm3_0b = _mm512_set_epi32(13 + 16, 10 + 16, 7 + 16, 4 + 16, 1 + 16, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
__m512i perm3_1 = _mm512_set_epi32(14 + 16,
11 + 16,
8 + 16,
5 + 16,
2 + 16,
15 + 16,
12 + 16,
9 + 16,
6 + 16,
3 + 16,
0 + 16,
13,
10,
7,
4,
1);
__m512i perm3_1b = _mm512_set_epi32(14 + 16, 11 + 16, 8 + 16, 5 + 16, 2 + 16, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
__m512i perm3_2 = _mm512_set_epi32(15 + 16,
12 + 16,
9 + 16,
6 + 16,
3 + 16,
0 + 16,
13 + 16,
10 + 16,
7 + 16,
4 + 16,
1 + 16,
14,
11,
8,
5,
2);
__m512i perm3_2b = _mm512_set_epi32(15 + 16, 12 + 16, 9 + 16, 6 + 16, 3 + 16, 0 + 16, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
int i, n;
for (i = 0, n = 0; i < n_symbs >> 4; i += 3, n++) {
i0 = ((__m512i *)mod_symbs[0])[i];
i1 = ((__m512i *)mod_symbs[0])[i + 1];
i2 = ((__m512i *)mod_symbs[0])[i + 2];
d0 = _mm512_permutex2var_epi32(i0, perm3_0, i1);
((__m512i *)tx_layers[0])[n] = _mm512_permutex2var_epi32(d0, perm3_0b, i2); // 11000000
d0 = _mm512_permutex2var_epi32(i0, perm3_1, i1);
((__m512i *)tx_layers[1])[n] = _mm512_permutex2var_epi32(d0, perm3_1b, i2); // 11000000
d0 = _mm512_permutex2var_epi32(i0, perm3_2, i1);
((__m512i *)tx_layers[2])[n] = _mm512_permutex2var_epi32(d0, perm3_2b, i2); // 11000000
}
#elif defined(__aarch64__)
for (int i = 0; i < n_symbs; i += 3) {
tx_layers[0][i / 3] = mod_symbs[0][i];
tx_layers[1][i / 3] = mod_symbs[0][i + 1];
tx_layers[2][i / 3] = mod_symbs[0][i + 2];
}
#else
simde__m256i i0, i1, i2, d0, d1, d2, d3;
simde__m256i perm3_0 = simde_mm256_set_epi32(5, 2, 7, 4, 1, 6, 3, 0);
simde__m256i perm3_1 = simde_mm256_set_epi32(6, 3, 0, 5, 2, 7, 4, 1);
simde__m256i perm3_2 = simde_mm256_set_epi32(7, 4, 1, 6, 3, 0, 5, 2);
int i, n;
for (i = 0, n = 0; i < n_symbs >> 3; i += 3, n++) {
i0 = ((simde__m256i *)mod_symbs[0])[i];
i1 = ((simde__m256i *)mod_symbs[0])[i + 1];
i2 = ((simde__m256i *)mod_symbs[0])[i + 2];
d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_0);
d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_0);
d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_0);
d3 = simde_mm256_blend_epi32(d0, d1, 0x38); // 00111000
((simde__m256i *)tx_layers[0])[n] = simde_mm256_blend_epi32(d3, d2, 0xc0); // 11000000
d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_1);
d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_1);
d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_1);
d3 = simde_mm256_blend_epi32(d0, d1, 0x18); // 00011000
((simde__m256i *)tx_layers[1])[n] = simde_mm256_blend_epi32(d3, d2, 0xe0); // 11100000
d0 = simde_mm256_permutevar8x32_epi32(i0, perm3_2);
d1 = simde_mm256_permutevar8x32_epi32(i1, perm3_2);
d2 = simde_mm256_permutevar8x32_epi32(i2, perm3_2);
d3 = simde_mm256_blend_epi32(d0, d1, 0x1c); // 00011100
((simde__m256i *)tx_layers[2])[n] = simde_mm256_blend_epi32(d3, d2, 0xe0); // 11100000
#ifdef DEBUG_LAYER_MAPPING
printf("\nsymb %d/%d\n", i << 3, n_symbs);
printf(" layer 0:\t");
for (int j = 0; j < 8 * 6; j += 6) {
printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
}
printf("\n layer 1:\t");
for (int j = 2; j < 8 * 6; j += 6) {
printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
}
printf("\n layer 2:\t");
for (int j = 4; j < 8 * 6; j += 6) {
printf("%d %d ", ((int16_t *)&mod_symbs[0][i << 3])[j], ((int16_t *)&mod_symbs[0][i << 3])[j + 1]);
}
printf("\n Mapping layer 0:\t");
for (int j = 0; j < 16; j++) {
printf("%d ", ((int16_t *)&tx_layers[0][n << 3])[j]);
}
printf("\n Mapping layer 1:\t");
for (int j = 0; j < 16; j++) {
printf("%d ", ((int16_t *)&tx_layers[1][n << 3])[j]);
}
printf("\n Mapping layer 2:\t");
for (int j = 0; j < 16; j++) {
printf("%d ", ((int16_t *)&tx_layers[2][n << 3])[j]);
}
#endif
}
if (i << 3 != n_symbs) {
for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 3) {
tx_layers[0][i2 / 3] = mod_symbs[0][i2];
tx_layers[1][i2 / 3] = mod_symbs[0][i2 + 1];
tx_layers[2][i2 / 3] = mod_symbs[0][i2 + 2];
}
}
#endif
} break;
case 4: {
#if defined(__AVX512BW__)
__m512i perm4 = _mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);
__m512i e;
int i;
for (i = 0; i < n_symbs >> 4; i++) {
e = _mm512_permutexvar_epi32(perm4, ((__m512i *)mod_symbs[0])[i]);
((__m128i *)tx_layers[0])[i] = _mm512_extracti64x2_epi64(e, 0);
((__m128i *)tx_layers[1])[i] = _mm512_extracti64x2_epi64(e, 1);
((__m128i *)tx_layers[2])[i] = _mm512_extracti64x2_epi64(e, 2);
((__m128i *)tx_layers[3])[i] = _mm512_extracti64x2_epi64(e, 3);
}
if (i << 4 != n_symbs) {
for (int i2 = ((n_symbs >> 4) << 4); i2 < n_symbs; i2 += 4) {
tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
tx_layers[2][i2 >> 2] = mod_symbs[0][i2 + 2];
tx_layers[3][i2 >> 2] = mod_symbs[0][i2 + 3];
}
}
#elif defined(__aarch64__)
int i;
#ifdef USE_NEON
// SIMDe doesn't handle this properly, gcc up to 14.2 neither
uint32x4_t d4;
for (i = 0; i < n_symbs >> 2; i++) {
d4 = ((uint32x4_t *)mod_symbs[0])[i];
((uint32_t *)tx_layers[0])[i] = vgetq_lane_u32(d4, 0);
((uint32_t *)tx_layers[1])[i] = vgetq_lane_u32(d4, 1);
((uint32_t *)tx_layers[2])[i] = vgetq_lane_u32(d4, 0);
((uint32_t *)tx_layers[3])[i] = vgetq_lane_u32(d4, 1);
}
if (i << 2 != n_symbs) {
for (int i2 = ((n_symbs >> 2) << 2); i2 < n_symbs; i2 += 4) {
tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
tx_layers[0][i2 >> 2] = mod_symbs[0][i2];
tx_layers[1][i2 >> 2] = mod_symbs[0][i2 + 1];
}
}
#else
for (int i = 0; i < n_symbs; i += 4) {
tx_layers[0][i >> 2] = mod_symbs[0][i];
tx_layers[1][i >> 2] = mod_symbs[0][i + 1];
tx_layers[2][i >> 2] = mod_symbs[0][i + 2];
tx_layers[3][i >> 2] = mod_symbs[0][i + 3];
}
#endif
#else
simde__m256i perm4 = simde_mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
simde__m256i e;
int i;
for (i = 0; i < n_symbs >> 3; i++) {
e = simde_mm256_permutevar8x32_epi32(((simde__m256i *)mod_symbs[0])[i], perm4);
((uint64_t *)tx_layers[0])[i] = simde_mm256_extract_epi64(e, 0);
((uint64_t *)tx_layers[1])[i] = simde_mm256_extract_epi64(e, 1);
((uint64_t *)tx_layers[2])[i] = simde_mm256_extract_epi64(e, 2);
((uint64_t *)tx_layers[3])[i] = simde_mm256_extract_epi64(e, 3);
}
if (i << 3 != n_symbs) {
for (int i2 = ((n_symbs >> 3) << 3); i2 < n_symbs; i2 += 3) {
tx_layers[0][i2 >> 1] = mod_symbs[0][i2];
tx_layers[1][i2 >> 1] = mod_symbs[0][i2 + 1];
tx_layers[2][i2 >> 1] = mod_symbs[0][i2 + 2];
tx_layers[3][i2 >> 1] = mod_symbs[0][i2 + 2];
}
}
#endif
} break;
case 5: case 5:
if (layer < 2) case 6:
for (int i = 0; i < n_symbs; i += 2) { case 7:
case 8:
/*
// Layer 0,1
for (int i = 0; i < n_symbs; i += 2) {
const int txIdx = i / 2; const int txIdx = i / 2;
tx_layer[txIdx] = mod_symbs[0][i + layer]; tx_layer[0][txIdx] = mod_symbs[0][i];
} tx_layer[1][txIdx] = mod_symbs[0][i + 1];
}
// layers 2,3,4
else else
for (int i = 0; i < n_symbs; i += 3) { for (int i = 0; i < n_symbs; i += 3) {
const int txIdx = i / 3; const int txIdx = i / 3;
tx_layer[txIdx] = mod_symbs[1][i + layer]; tx_layer[2][txIdx] = mod_symbs[1][i + 2];
tx_layer[3][txIdx] = mod_symbs[1][i + 3];
tx_layer[4][txIdx] = mod_symbs[1][i + 4];
} }
break; break;
...@@ -288,7 +546,12 @@ void nr_layer_mapping(int nbCodes, ...@@ -288,7 +546,12 @@ void nr_layer_mapping(int nbCodes,
for (int q=0; q<2; q++) for (int q=0; q<2; q++)
for (int i = 0; i < n_symbs; i += 3) { for (int i = 0; i < n_symbs; i += 3) {
const int txIdx = i / 3; const int txIdx = i / 3;
tx_layer[txIdx] = mod_symbs[q][i + layer]; tx_layer[0][txIdx] = mod_symbs[q][i + layer];
tx_layer[1][txIdx] = mod_symbs[q][i + layer];
tx_layer[2][txIdx] = mod_symbs[q][i + layer];
tx_layer[3][txIdx] = mod_symbs[q][i + layer];
tx_layer[4][txIdx] = mod_symbs[q][i + layer];
tx_layer[5][txIdx] = mod_symbs[q][i + layer];
} }
break; break;
...@@ -312,7 +575,7 @@ void nr_layer_mapping(int nbCodes, ...@@ -312,7 +575,7 @@ void nr_layer_mapping(int nbCodes,
tx_layer[txIdx] = mod_symbs[q][i + layer]; tx_layer[txIdx] = mod_symbs[q][i + layer];
} }
break; break;
*/
default: default:
AssertFatal(0, "Invalid number of layers %d\n", n_layers); AssertFatal(0, "Invalid number of layers %d\n", n_layers);
} }
...@@ -337,7 +600,7 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH) ...@@ -337,7 +600,7 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH)
simde__m128i norm128; simde__m128i norm128;
if ((Msc_PUSCH % 1536) > 0) { if ((Msc_PUSCH % 1536) > 0) {
for (i = 0, ip = 0; i < Msc_PUSCH; i++, ip+=4) { for (i = 0, ip = 0; i < Msc_PUSCH; i++, ip += 4) {
dft_in0[ip] = d[i]; dft_in0[ip] = d[i];
} }
} }
...@@ -357,19 +620,18 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH) ...@@ -357,19 +620,18 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH)
} }
if ((Msc_PUSCH % 1536) > 0) { if ((Msc_PUSCH % 1536) > 0) {
for (i = 0, ip = 0; i < Msc_PUSCH; i++, ip+=4) for (i = 0, ip = 0; i < Msc_PUSCH; i++, ip += 4)
z[i] = dft_out0[ip]; z[i] = dft_out0[ip];
} }
} }
void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation) void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation)
{ {
const int nsymb = fp->symbols_per_slot * fp->slots_per_frame/10; const int nsymb = fp->symbols_per_slot * fp->slots_per_frame / 10;
const double Tc=(1/480e3/4096); const double Tc = (1 / 480e3 / 4096);
const double Nu=2048*64*(1/(float)(1<<fp->numerology_index)); const double Nu = 2048 * 64 * (1 / (float)(1 << fp->numerology_index));
const double Ncp0=16*64 + (144*64*(1/(float)(1<<fp->numerology_index))); const double Ncp0 = 16 * 64 + (144 * 64 * (1 / (float)(1 << fp->numerology_index)));
const double Ncp1=(144*64*(1/(float)(1<<fp->numerology_index))); const double Ncp1 = (144 * 64 * (1 / (float)(1 << fp->numerology_index)));
LOG_D(PHY, "Doing symbol rotation calculation for TX/RX, f0 %f Hz, Nsymb %d\n", f0, nsymb); LOG_D(PHY, "Doing symbol rotation calculation for TX/RX, f0 %f Hz, Nsymb %d\n", f0, nsymb);
...@@ -430,7 +692,9 @@ void init_timeshift_rotation(NR_DL_FRAME_PARMS *fp) ...@@ -430,7 +692,9 @@ void init_timeshift_rotation(NR_DL_FRAME_PARMS *fp)
fp->timeshift_symbol_rotation[i].i = (int16_t)round(exp_im * 32767); fp->timeshift_symbol_rotation[i].i = (int16_t)round(exp_im * 32767);
if (i < 10) if (i < 10)
LOG_D(PHY,"Timeshift symbol rotation %d => (%d,%d) %f\n",i, LOG_D(PHY,
"Timeshift symbol rotation %d => (%d,%d) %f\n",
i,
fp->timeshift_symbol_rotation[i].r, fp->timeshift_symbol_rotation[i].r,
fp->timeshift_symbol_rotation[i].i, fp->timeshift_symbol_rotation[i].i,
poff); poff);
...@@ -441,13 +705,13 @@ c16_t nr_layer_precoder(int sz, c16_t datatx_F_precoding[][sz], const char *prec ...@@ -441,13 +705,13 @@ c16_t nr_layer_precoder(int sz, c16_t datatx_F_precoding[][sz], const char *prec
{ {
c16_t precodatatx_F = {0}; c16_t precodatatx_F = {0};
for (int al = 0; al<n_layers; al++) { for (int al = 0; al < n_layers; al++) {
c16_t antenna = datatx_F_precoding[al][re_offset]; c16_t antenna = datatx_F_precoding[al][re_offset];
switch (prec_matrix[al]) { switch (prec_matrix[al]) {
case '0': //multiply by zero case '0': // multiply by zero
break; break;
case '1': //multiply by 1 case '1': // multiply by 1
precodatatx_F = c16add(precodatatx_F, antenna); precodatatx_F = c16add(precodatatx_F, antenna);
break; break;
...@@ -492,15 +756,15 @@ c16_t nr_layer_precoder_cm(int n_layers, ...@@ -492,15 +756,15 @@ c16_t nr_layer_precoder_cm(int n_layers,
} }
void nr_layer_precoder_simd(const int n_layers, void nr_layer_precoder_simd(const int n_layers,
const int n_symbols, const int n_symbols,
const int symSz, const int symSz,
const c16_t txdataF_res_mapped[n_layers][n_symbols][symSz], const c16_t txdataF_res_mapped[n_layers][n_symbols][symSz],
const int ant, const int ant,
const nfapi_nr_pm_pdu_t *pmi_pdu, const nfapi_nr_pm_pdu_t *pmi_pdu,
const int symbol, const int symbol,
const int sc_offset, const int sc_offset,
const int re_cnt, const int re_cnt,
c16_t *txdataF_precoded) c16_t *txdataF_precoded)
{ {
uint32_t sc = sc_offset; uint32_t sc = sc_offset;
c16_t prec_weight = {0}; c16_t prec_weight = {0};
...@@ -510,22 +774,25 @@ void nr_layer_precoder_simd(const int n_layers, ...@@ -510,22 +774,25 @@ void nr_layer_precoder_simd(const int n_layers,
// 256 SIMD: Do 8 RE in one iteration, 3 iterations for 2 RB // 256 SIMD: Do 8 RE in one iteration, 3 iterations for 2 RB
#ifdef __AVX2__ #ifdef __AVX2__
const uint32_t re_cnt_align8 = re_cnt & ~7; const uint32_t re_cnt_align8 = re_cnt & ~7;
for(; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) {
// Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1]
for(int nl = 0; nl < n_layers; nl++) { for (int nl = 0; nl < n_layers; nl++) {
prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re;
prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im;
const simde__m256i x = simde_mm256_loadu_si256(&txdataF_res_mapped[nl][symbol][sc]); const simde__m256i x = simde_mm256_loadu_si256(&txdataF_res_mapped[nl][symbol][sc]);
// Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size
const simde__m256i w_c = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w const simde__m256i w_c = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w
const simde__m256i w_s = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w const simde__m256i w_s = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w
// Multiplication and shift // Multiplication and shift
const simde__m256i reals = simde_mm256_srai_epi32(simde_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 const simde__m256i reals =
const simde__m256i imags = simde_mm256_slli_epi32(simde_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 simde_mm256_srai_epi32(simde_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
const simde__m256i imags = simde_mm256_slli_epi32(
simde_mm256_madd_epi16(x, w_s),
1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
// Re-arrange to match c16_t format // Re-arrange to match c16_t format
const simde__m256i produ = simde_mm256_blend_epi16(reals, imags, 0xAA); const simde__m256i produ = simde_mm256_blend_epi16(reals, imags, 0xAA);
...@@ -540,43 +807,40 @@ void nr_layer_precoder_simd(const int n_layers, ...@@ -540,43 +807,40 @@ void nr_layer_precoder_simd(const int n_layers,
// 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB
const uint32_t re_cnt_align4 = re_cnt & ~3; const uint32_t re_cnt_align4 = re_cnt & ~3;
for(; sc < sc_offset+re_cnt_align4; sc += sizeof(simde__m128i) / sizeof(prec_weight)) { for (; sc < sc_offset + re_cnt_align4; sc += sizeof(simde__m128i) / sizeof(prec_weight)) {
#ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed
c16_t y_triv[4]; c16_t y_triv[4];
for(int i = 0; i < 4; i++) for (int i = 0; i < 4; i++)
y_triv[i] = nr_layer_precoder_cm(n_layers, y_triv[i] = nr_layer_precoder_cm(n_layers, NR_SYMBOLS_PER_SLOT, symSz, txdataF_res_mapped, ant, pmi_pdu, symbol, sc + i);
NR_SYMBOLS_PER_SLOT, memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv));
symSz, #endif
txdataF_res_mapped,
ant,
pmi_pdu,
symbol,
sc + i);
memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv));
#endif
// Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4) // Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4)
simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1]
for(int nl = 0; nl < n_layers; nl++) { for (int nl = 0; nl < n_layers; nl++) {
prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re;
prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im;
const simde__m128i x = simde_mm_loadu_si128(&txdataF_res_mapped[nl][symbol][sc]); const simde__m128i x = simde_mm_loadu_si128(&txdataF_res_mapped[nl][symbol][sc]);
// Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size
const simde__m128i w_c = simde_mm_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w const simde__m128i w_c = simde_mm_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w
const simde__m128i w_s = simde_mm_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w const simde__m128i w_s = simde_mm_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w
// Multiplication and shift // Multiplication and shift
const simde__m128i reals = simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 const simde__m128i reals =
const simde__m128i imags = simde_mm_slli_epi32(simde_mm_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15
const simde__m128i imags = simde_mm_slli_epi32(
simde_mm_madd_epi16(x, w_s),
1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16
/* Re-arrange to match c16_t format /* Re-arrange to match c16_t format
bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 | 112 bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 |
reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] | R3.r[31] (0)*15} 112 reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 |
imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]| R2.i[15..30] | 0 R3.i[0..14]| R3.i[15..30] } R3.r[15..30] | R3.r[31] (0)*15} imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]|
16b from {reals | imags | reals | imags | reals | imags | reals | imags } R2.i[15..30] | 0 R3.i[0..14]| R3.i[15..30] } 16b from {reals | imags | reals | imags | reals
produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30] | R2.r[15..30] | R2.i[15..30] | R3.r[15..30] | R3.i[15..30] } | imags | reals | imags } produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30]
| R2.r[15..30] | R2.i[15..30] | R3.r[15..30] | R3.i[15..30] }
*/ */
const simde__m128i produ = simde_mm_blend_epi16(reals, imags, 0xAA); const simde__m128i produ = simde_mm_blend_epi16(reals, imags, 0xAA);
...@@ -586,30 +850,30 @@ void nr_layer_precoder_simd(const int n_layers, ...@@ -586,30 +850,30 @@ void nr_layer_precoder_simd(const int n_layers,
// Store the result to txdataF // Store the result to txdataF
simde_mm_storeu_si128(&txdataF_precoded[sc], y); simde_mm_storeu_si128(&txdataF_precoded[sc], y);
#ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Print simd and trivial result, TODO: To be removed #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Print simd and trivial result, TODO: To be removed
c16_t *y_simd = (c16_t*) &y; c16_t *y_simd = (c16_t *)&y;
printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n",
re_cnt, re_cnt,
sc, sc,
y_simd[0].r, y_simd[0].r,
y_simd[0].i, y_simd[0].i,
y_simd[1].r, y_simd[1].r,
y_simd[1].i, y_simd[1].i,
y_simd[2].r, y_simd[2].r,
y_simd[2].i, y_simd[2].i,
y_simd[3].r, y_simd[3].r,
y_simd[3].i); y_simd[3].i);
printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n",
re_cnt, re_cnt,
sc, sc,
y_triv[0].r, y_triv[0].r,
y_triv[0].i, y_triv[0].i,
y_triv[1].r, y_triv[1].r,
y_triv[1].i, y_triv[1].i,
y_triv[2].r, y_triv[2].r,
y_triv[2].i, y_triv[2].i,
y_triv[3].r, y_triv[3].r,
y_triv[3].i); y_triv[3].i);
#endif #endif
} }
} }
...@@ -60,8 +60,7 @@ void nr_layer_mapping(int nbCodes, ...@@ -60,8 +60,7 @@ void nr_layer_mapping(int nbCodes,
uint8_t n_layers, uint8_t n_layers,
int layerSz, int layerSz,
uint32_t n_symbs, uint32_t n_symbs,
c16_t tx_layers[layerSz], c16_t tx_layers[][layerSz]);
int l);
/*! \brief Perform NR layer mapping. TS 38.211 V15.4.0 subclause 7.3.1.3 /*! \brief Perform NR layer mapping. TS 38.211 V15.4.0 subclause 7.3.1.3
@param[in] ulsch_ue, double Pointer to NR_UE_ULSCH_t struct @param[in] ulsch_ue, double Pointer to NR_UE_ULSCH_t struct
......
...@@ -248,9 +248,11 @@ void nr_generate_dci_top(processingData_L1tx_t *msgTx, int slot, int txdataF_off ...@@ -248,9 +248,11 @@ void nr_generate_dci_top(processingData_L1tx_t *msgTx, int slot, int txdataF_off
{ {
PHY_VARS_gNB *gNB = msgTx->gNB; PHY_VARS_gNB *gNB = msgTx->gNB;
NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms; NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
start_meas(&gNB->dci_generation_stats);
for (int i = 0; i < msgTx->num_ul_pdcch; i++) for (int i = 0; i < msgTx->num_ul_pdcch; i++)
nr_generate_dci(msgTx->gNB, &msgTx->ul_pdcch_pdu[i].pdcch_pdu.pdcch_pdu_rel15, txdataF_offset, frame_parms, slot); nr_generate_dci(msgTx->gNB, &msgTx->ul_pdcch_pdu[i].pdcch_pdu.pdcch_pdu_rel15, txdataF_offset, frame_parms, slot);
for (int i = 0; i < msgTx->num_dl_pdcch; i++) for (int i = 0; i < msgTx->num_dl_pdcch; i++)
nr_generate_dci(msgTx->gNB, &msgTx->pdcch_pdu[i].pdcch_pdu_rel15, txdataF_offset, frame_parms, slot); nr_generate_dci(msgTx->gNB, &msgTx->pdcch_pdu[i].pdcch_pdu_rel15, txdataF_offset, frame_parms, slot);
stop_meas(&gNB->dci_generation_stats);
} }
This source diff could not be displayed because it is too large. You can view the blob instead.
...@@ -520,6 +520,8 @@ typedef struct PHY_VARS_gNB_s { ...@@ -520,6 +520,8 @@ typedef struct PHY_VARS_gNB_s {
time_stats_t dlsch_interleaving_stats; time_stats_t dlsch_interleaving_stats;
time_stats_t dlsch_segmentation_stats; time_stats_t dlsch_segmentation_stats;
time_stats_t dci_generation_stats;
time_stats_t phase_comp_stats;
time_stats_t rx_pusch_stats; time_stats_t rx_pusch_stats;
time_stats_t rx_pusch_init_stats; time_stats_t rx_pusch_init_stats;
time_stats_t rx_pusch_symbol_processing_stats; time_stats_t rx_pusch_symbol_processing_stats;
......
...@@ -290,6 +290,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx, ...@@ -290,6 +290,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
//apply the OFDM symbol rotation here //apply the OFDM symbol rotation here
if (gNB->phase_comp) { if (gNB->phase_comp) {
start_meas(&gNB->phase_comp_stats);
for(int i = 0; i < gNB->common_vars.num_beams_period; ++i) { for(int i = 0; i < gNB->common_vars.num_beams_period; ++i) {
for (int aa = 0; aa < cfg->carrier_config.num_tx_ant.value; aa++) { for (int aa = 0; aa < cfg->carrier_config.num_tx_ant.value; aa++) {
apply_nr_rotation_TX(fp, apply_nr_rotation_TX(fp,
...@@ -304,6 +305,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx, ...@@ -304,6 +305,7 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
T_INT(aa), T_BUFFER(&gNB->common_vars.txdataF[aa][txdataF_offset], fp->samples_per_slot_wCP*sizeof(int32_t))); T_INT(aa), T_BUFFER(&gNB->common_vars.txdataF[aa][txdataF_offset], fp->samples_per_slot_wCP*sizeof(int32_t)));
} }
} }
stop_meas(&gNB->phase_comp_stats);
} }
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX + gNB->CC_id, 0); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX + gNB->CC_id, 0);
......
...@@ -960,10 +960,12 @@ printf("%d\n", slot); ...@@ -960,10 +960,12 @@ printf("%d\n", slot);
reset_meas(&gNB->dlsch_segmentation_stats); reset_meas(&gNB->dlsch_segmentation_stats);
reset_meas(&gNB->dlsch_modulation_stats); reset_meas(&gNB->dlsch_modulation_stats);
reset_meas(&gNB->dlsch_encoding_stats); reset_meas(&gNB->dlsch_encoding_stats);
reset_meas(&gNB->dci_generation_stats);
reset_meas(&gNB->tinput); reset_meas(&gNB->tinput);
reset_meas(&gNB->tprep); reset_meas(&gNB->tprep);
reset_meas(&gNB->tparity); reset_meas(&gNB->tparity);
reset_meas(&gNB->toutput); reset_meas(&gNB->toutput);
reset_meas(&gNB->phase_comp_stats);
uint32_t errors_scrambling[16] = {0}; uint32_t errors_scrambling[16] = {0};
int n_errors[16] = {0}; int n_errors[16] = {0};
...@@ -1259,6 +1261,7 @@ printf("%d\n", slot); ...@@ -1259,6 +1261,7 @@ printf("%d\n", slot);
UE->dl_harq_processes[0][slot].C, UE->dl_harq_processes[0][slot].C,
msgDataTx->dlsch[0][0].harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0] << 3); msgDataTx->dlsch[0][0].harq_process.pdsch_pdu.pdsch_pdu_rel15.TBSize[0] << 3);
printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx"); printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");
printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time"); printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time");
printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time"); printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time");
printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time"); printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time");
...@@ -1271,6 +1274,8 @@ printf("%d\n", slot); ...@@ -1271,6 +1274,8 @@ printf("%d\n", slot);
printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time"); printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time");
printStatIndent2(&gNB->dlsch_resource_mapping_stats, "DLSCH Resource Mapping time"); printStatIndent2(&gNB->dlsch_resource_mapping_stats, "DLSCH Resource Mapping time");
printStatIndent2(&gNB->dlsch_precoding_stats,"DLSCH Layer Precoding time"); printStatIndent2(&gNB->dlsch_precoding_stats,"DLSCH Layer Precoding time");
if (gNB->phase_comp)
printStatIndent2(&gNB->phase_comp_stats, "Phase Compensation");
printf("\nUE function statistics (per %d us slot)\n", 1000 >> *scc->ssbSubcarrierSpacing); printf("\nUE function statistics (per %d us slot)\n", 1000 >> *scc->ssbSubcarrierSpacing);
for (int i = RX_PDSCH_STATS; i <= DLSCH_PROCEDURES_STATS; i++) { for (int i = RX_PDSCH_STATS; i <= DLSCH_PROCEDURES_STATS; i++) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment