Commit 6fcfbc49 authored by Jaroslava Fiedlerova's avatar Jaroslava Fiedlerova

T2 - offload TB for encoding, decoding - on gNB side only, do CRC check on the T2

parent 21172d27
...@@ -56,7 +56,7 @@ ...@@ -56,7 +56,7 @@
// this socket is the NUMA socket, so the hardware CPU id (numa is complex) // this socket is the NUMA socket, so the hardware CPU id (numa is complex)
#define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
#define MAX_QUEUES 16 #define MAX_QUEUES 32
#define OPS_CACHE_SIZE 256U #define OPS_CACHE_SIZE 256U
#define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
...@@ -121,7 +121,7 @@ struct test_op_params { ...@@ -121,7 +121,7 @@ struct test_op_params {
uint16_t num_lcores; uint16_t num_lcores;
int vector_mask; int vector_mask;
rte_atomic16_t sync; rte_atomic16_t sync;
struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][32];
}; };
/* Contains per lcore params */ /* Contains per lcore params */
...@@ -181,6 +181,7 @@ optimal_mempool_size(unsigned int val) ...@@ -181,6 +181,7 @@ optimal_mempool_size(unsigned int val)
static int create_mempools(struct active_device *ad, int socket_id, uint16_t num_ops, int out_buff_sz, int in_max_sz) static int create_mempools(struct active_device *ad, int socket_id, uint16_t num_ops, int out_buff_sz, int in_max_sz)
{ {
unsigned int ops_pool_size, mbuf_pool_size, data_room_size = 0; unsigned int ops_pool_size, mbuf_pool_size, data_room_size = 0;
num_ops = 1;
uint8_t nb_segments = 1; uint8_t nb_segments = 1;
ops_pool_size = optimal_mempool_size(RTE_MAX( ops_pool_size = optimal_mempool_size(RTE_MAX(
/* Ops used plus 1 reference op */ /* Ops used plus 1 reference op */
...@@ -374,26 +375,24 @@ static int add_dev(uint8_t dev_id, struct rte_bbdev_info *info) ...@@ -374,26 +375,24 @@ static int add_dev(uint8_t dev_id, struct rte_bbdev_info *info)
// DPDK BBDEV modified - nb_segments used, we are not using struct op_data_entries *ref_entries, but struct rte_mbuf *m_head, // DPDK BBDEV modified - nb_segments used, we are not using struct op_data_entries *ref_entries, but struct rte_mbuf *m_head,
// rte_pktmbuf_reset(m_head) added? if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) -> no code in else? // rte_pktmbuf_reset(m_head) added? if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) -> no code in else?
static int init_op_data_objs(struct rte_bbdev_op_data *bufs, static int init_op_data_objs_dec(struct rte_bbdev_op_data *bufs,
uint8_t *input, uint8_t *input,
uint32_t data_len, t_nrLDPCoffload_params *offloadParams,
struct rte_mbuf *m_head, struct rte_mempool *mbuf_pool,
struct rte_mempool *mbuf_pool, const uint16_t n,
const uint16_t n, enum op_data_type op_type,
enum op_data_type op_type, uint16_t min_alignment)
uint16_t min_alignment)
{ {
int ret; uint32_t data_len;
unsigned int i, j;
bool large_input = false; bool large_input = false;
uint8_t nb_segments = 1; for (int i = 0; i < n; ++i) {
for (i = 0; i < n; ++i) { data_len = offloadParams->E_cb[i];
char *data; char *data;
struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
TEST_ASSERT_NOT_NULL(m_head, "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
op_type, n, mbuf_pool->size);
if (data_len > RTE_BBDEV_LDPC_E_MAX_MBUF) { if (data_len > RTE_BBDEV_LDPC_E_MAX_MBUF) {
/*
* Special case when DPDK mbuf cannot handle
* the required input size
*/
printf("Warning: Larger input size than DPDK mbuf %u\n", data_len); printf("Warning: Larger input size than DPDK mbuf %u\n", data_len);
large_input = true; large_input = true;
} }
...@@ -406,70 +405,80 @@ static int init_op_data_objs(struct rte_bbdev_op_data *bufs, ...@@ -406,70 +405,80 @@ static int init_op_data_objs(struct rte_bbdev_op_data *bufs,
/* Allocate a fake overused mbuf */ /* Allocate a fake overused mbuf */
data = rte_malloc(NULL, data_len, 0); data = rte_malloc(NULL, data_len, 0);
TEST_ASSERT_NOT_NULL(data, "rte malloc failed with %u bytes", data_len); TEST_ASSERT_NOT_NULL(data, "rte malloc failed with %u bytes", data_len);
memcpy(data, input, data_len); memcpy(data, &input[i * LDPC_MAX_CB_SIZE], data_len);
m_head->buf_addr = data; m_head->buf_addr = data;
m_head->buf_iova = rte_malloc_virt2iova(data); m_head->buf_iova = rte_malloc_virt2iova(data);
m_head->data_off = 0; m_head->data_off = 0;
m_head->data_len = data_len; m_head->data_len = data_len;
} else { } else {
// rte_pktmbuf_reset added
rte_pktmbuf_reset(m_head); rte_pktmbuf_reset(m_head);
data = rte_pktmbuf_append(m_head, data_len); data = rte_pktmbuf_append(m_head, data_len);
TEST_ASSERT_NOT_NULL(data, "Couldn't append %u bytes to mbuf from %d data type mbuf pool", data_len, op_type); TEST_ASSERT_NOT_NULL(data, "Couldn't append %u bytes to mbuf from %d data type mbuf pool", data_len, op_type);
TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
"Data addr in mbuf (%p) is not aligned to device min alignment (%u)", "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
data, data,
min_alignment); min_alignment);
rte_memcpy(data, input, data_len); rte_memcpy(data, &input[i * LDPC_MAX_CB_SIZE], data_len);
} }
bufs[i].length += data_len; bufs[i].length += data_len;
}
}
return 0;
}
for (j = 1; j < nb_segments; ++j) { static int init_op_data_objs_enc(struct rte_bbdev_op_data *bufs,
struct rte_mbuf *m_tail = rte_pktmbuf_alloc(mbuf_pool); uint8_t **input_enc,
TEST_ASSERT_NOT_NULL(m_tail, t_nrLDPCoffload_params *offloadParams,
"Not enough mbufs in %d data type mbuf pool (needed %d, available %u)", struct rte_mbuf *m_head,
op_type, struct rte_mempool *mbuf_pool,
n * nb_segments, const uint16_t n,
mbuf_pool->size); enum op_data_type op_type,
uint16_t min_alignment)
{
uint32_t data_len;
bool large_input = false;
for (int i = 0; i < n; ++i) {
data_len = offloadParams->Kr;
char *data;
struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
TEST_ASSERT_NOT_NULL(m_head, "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
op_type, n, mbuf_pool->size);
data = rte_pktmbuf_append(m_tail, data_len); if (data_len > RTE_BBDEV_LDPC_E_MAX_MBUF) {
TEST_ASSERT_NOT_NULL(data, "Couldn't append %u bytes to mbuf from %d data type mbuf pool", data_len, op_type); printf("Warning: Larger input size than DPDK mbuf %u\n", data_len);
large_input = true;
}
bufs[i].data = m_head;
bufs[i].offset = 0;
bufs[i].length = 0;
if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {
if ((op_type == DATA_INPUT) && large_input) {
/* Allocate a fake overused mbuf */
data = rte_malloc(NULL, data_len, 0);
TEST_ASSERT_NOT_NULL(data, "rte malloc failed with %u bytes", data_len);
memcpy(data, &input_enc[0], data_len);
m_head->buf_addr = data;
m_head->buf_iova = rte_malloc_virt2iova(data);
m_head->data_off = 0;
m_head->data_len = data_len;
} else {
rte_pktmbuf_reset(m_head);
data = rte_pktmbuf_append(m_head, data_len);
TEST_ASSERT_NOT_NULL(data, "Couldn't append %u bytes to mbuf from %d data type mbuf pool", data_len, op_type);
TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
"Data addr in mbuf (%p) is not aligned to device min alignment (%u)", "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
data, data,
min_alignment); min_alignment);
rte_memcpy(data, input, data_len); rte_memcpy(data, input_enc[i], data_len);
bufs[i].length += data_len;
ret = rte_pktmbuf_chain(m_head, m_tail);
TEST_ASSERT_SUCCESS(ret, "Couldn't chain mbufs from %d data type mbuf pool", op_type);
} }
} else { bufs[i].length += data_len;
/* allocate chained-mbuf for output buffer */
/*for (j = 1; j < nb_segments; ++j) {
struct rte_mbuf *m_tail =
rte_pktmbuf_alloc(mbuf_pool);
TEST_ASSERT_NOT_NULL(m_tail,
"Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
op_type,
n * nb_segments,
mbuf_pool->size);
ret = rte_pktmbuf_chain(m_head, m_tail);
TEST_ASSERT_SUCCESS(ret,
"Couldn't chain mbufs from %d data type mbuf pool",
op_type);
}*/
} }
} }
return 0; return 0;
} }
// DPDK BBEV copy // DPDK BBEV copy
static int allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, const int socket) static int allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, const int socket)
{ {
...@@ -501,7 +510,7 @@ free_buffers(struct active_device *ad, struct test_op_params *op_params) ...@@ -501,7 +510,7 @@ free_buffers(struct active_device *ad, struct test_op_params *op_params)
rte_mempool_free(ad->harq_in_mbuf_pool); rte_mempool_free(ad->harq_in_mbuf_pool);
rte_mempool_free(ad->harq_out_mbuf_pool); rte_mempool_free(ad->harq_out_mbuf_pool);
for (int i = 0; i < rte_lcore_count(); ++i) { for (int i = 2; i < rte_lcore_count(); ++i) {
for (int j = 0; j < RTE_MAX_NUMA_NODES; ++j) { for (int j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
rte_free(op_params->q_bufs[j][i].inputs); rte_free(op_params->q_bufs[j][i].inputs);
rte_free(op_params->q_bufs[j][i].hard_outputs); rte_free(op_params->q_bufs[j][i].hard_outputs);
...@@ -525,14 +534,14 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, ...@@ -525,14 +534,14 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
t_nrLDPCoffload_params *p_offloadParams) t_nrLDPCoffload_params *p_offloadParams)
{ {
unsigned int i; unsigned int i;
for (i = 0; i < n; ++i) { for (i = 0; i < p_offloadParams->C; ++i) {
ops[i]->ldpc_dec.cb_params.e = p_offloadParams->E; ops[i]->ldpc_dec.cb_params.e = p_offloadParams->E_cb[i];
ops[i]->ldpc_dec.basegraph = p_offloadParams->BG; ops[i]->ldpc_dec.basegraph = p_offloadParams->BG;
ops[i]->ldpc_dec.z_c = p_offloadParams->Z; ops[i]->ldpc_dec.z_c = p_offloadParams->Z;
ops[i]->ldpc_dec.q_m = p_offloadParams->Qm; ops[i]->ldpc_dec.q_m = p_offloadParams->Qm;
ops[i]->ldpc_dec.n_filler = p_offloadParams->F; ops[i]->ldpc_dec.n_filler = p_offloadParams->F;
ops[i]->ldpc_dec.n_cb = p_offloadParams->n_cb; ops[i]->ldpc_dec.n_cb = p_offloadParams->n_cb;
ops[i]->ldpc_dec.iter_max = 20; ops[i]->ldpc_dec.iter_max = p_offloadParams->numMaxIter;
ops[i]->ldpc_dec.rv_index = p_offloadParams->rv; ops[i]->ldpc_dec.rv_index = p_offloadParams->rv;
ops[i]->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | ops[i]->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE | RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
...@@ -541,10 +550,14 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, ...@@ -541,10 +550,14 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
if (p_offloadParams->setCombIn) { if (p_offloadParams->setCombIn) {
ops[i]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; ops[i]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
} }
LOG_D(PHY,"ULSCH %02d HARQPID %02d R %02d COMBIN %d RV %d NCB %05d \n", ulsch_id, harq_pid, r, p_offloadParams->setCombIn, p_offloadParams->rv, p_offloadParams->n_cb); if (p_offloadParams->C > 1) {
ops[i]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP;
ops[i]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK;
}
//LOG_W(PHY,"ULSCH %02d HARQPID %02d R %02d COMBIN %d RV %d NCB %05d NUM OPS %d E %05d\n", ulsch_id, harq_pid, r, p_offloadParams->setCombIn, p_offloadParams->rv, p_offloadParams->n_cb, n, p_offloadParams->E_cb[i]);
ops[i]->ldpc_dec.code_block_mode = 1; // ldpc_dec->code_block_mode; ops[i]->ldpc_dec.code_block_mode = 1; // ldpc_dec->code_block_mode;
ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + r * LDPC_MAX_CB_SIZE; ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + i * LDPC_MAX_CB_SIZE;
ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + r * LDPC_MAX_CB_SIZE; ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id * 64 * LDPC_MAX_CB_SIZE + i * LDPC_MAX_CB_SIZE;
if (bufs->hard_outputs != NULL) if (bufs->hard_outputs != NULL)
ops[i]->ldpc_dec.hard_output = bufs->hard_outputs[start_idx + i]; ops[i]->ldpc_dec.hard_output = bufs->hard_outputs[start_idx + i];
if (bufs->inputs != NULL) if (bufs->inputs != NULL)
...@@ -568,7 +581,7 @@ static void set_ldpc_enc_op(struct rte_bbdev_enc_op **ops, ...@@ -568,7 +581,7 @@ static void set_ldpc_enc_op(struct rte_bbdev_enc_op **ops,
{ {
// struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; // struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc;
for (int i = 0; i < n; ++i) { for (int i = 0; i < n; ++i) {
ops[i]->ldpc_enc.cb_params.e = p_offloadParams->E; ops[i]->ldpc_enc.cb_params.e = p_offloadParams->E_cb[i];
ops[i]->ldpc_enc.basegraph = p_offloadParams->BG; ops[i]->ldpc_enc.basegraph = p_offloadParams->BG;
ops[i]->ldpc_enc.z_c = p_offloadParams->Z; ops[i]->ldpc_enc.z_c = p_offloadParams->Z;
ops[i]->ldpc_enc.q_m = p_offloadParams->Qm; ops[i]->ldpc_enc.q_m = p_offloadParams->Qm;
...@@ -591,33 +604,44 @@ static int retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, ...@@ -591,33 +604,44 @@ static int retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops,
uint8_t *p_out) uint8_t *p_out)
{ {
struct rte_bbdev_op_data *hard_output; struct rte_bbdev_op_data *hard_output;
uint16_t data_len = 0;
struct rte_mbuf *m; struct rte_mbuf *m;
unsigned int i; unsigned int i;
char *data; char *data;
int offset = 0;
for (i = 0; i < n; ++i) { for (i = 0; i < n; ++i) {
hard_output = &ops[i]->ldpc_dec.hard_output; hard_output = &ops[i]->ldpc_dec.hard_output;
m = hard_output->data; m = hard_output->data;
uint16_t data_len = rte_pktmbuf_data_len(m) - hard_output->offset; data_len = rte_pktmbuf_data_len(m) - hard_output->offset;
data = m->buf_addr; data = m->buf_addr;
memcpy(p_out, data + m->data_off, data_len); memcpy(&p_out[offset], data + m->data_off, data_len);
offset += data_len;
rte_pktmbuf_free(ops[i]->ldpc_dec.hard_output.data);
rte_pktmbuf_free(ops[i]->ldpc_dec.input.data);
} }
return 0; return 0;
} }
static int retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, struct rte_bbdev_enc_op *ref_op, uint8_t *p_out) static int retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, struct rte_bbdev_enc_op *ref_op, uint8_t *p_out, uint32_t *E)
{ {
struct rte_bbdev_op_data *output; struct rte_bbdev_op_data *output;
struct rte_mbuf *m; struct rte_mbuf *m;
unsigned int i; unsigned int i;
char *data; char *data;
uint8_t *out;
int offset = 0;
for (i = 0; i < n; ++i) { for (i = 0; i < n; ++i) {
output = &ops[i]->ldpc_enc.output; output = &ops[i]->ldpc_enc.output;
m = output->data; m = output->data;
uint16_t data_len = min((LDPC_MAX_CB_SIZE) / 8, rte_pktmbuf_data_len(m)); // fix me uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset;
out = &p_out[offset];
data = m->buf_addr; data = m->buf_addr;
for (int byte = 0; byte < data_len; byte++) for (int byte = 0; byte < data_len; byte++)
for (int bit = 0; bit < 8; bit++) for (int bit = 0; bit < 8; bit++)
p_out[byte * 8 + bit] = (data[m->data_off + byte] >> (7 - bit)) & 1; out[byte * 8 + bit] = (data[m->data_off + byte] >> (7 - bit)) & 1;
offset += E[i];
rte_pktmbuf_free(ops[i]->ldpc_enc.output.data);
rte_pktmbuf_free(ops[i]->ldpc_enc.input.data);
} }
return 0; return 0;
} }
...@@ -632,7 +656,7 @@ static int init_test_op_params(struct test_op_params *op_params, ...@@ -632,7 +656,7 @@ static int init_test_op_params(struct test_op_params *op_params,
{ {
int ret = 0; int ret = 0;
if (op_type == RTE_BBDEV_OP_LDPC_DEC) { if (op_type == RTE_BBDEV_OP_LDPC_DEC) {
ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, &op_params->ref_dec_op, 1); ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, &op_params->ref_dec_op, num_to_process);
op_params->mp_dec = ops_mp; op_params->mp_dec = ops_mp;
} else { } else {
ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, &op_params->ref_enc_op, 1); ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, &op_params->ref_enc_op, 1);
...@@ -654,8 +678,8 @@ pmd_lcore_ldpc_dec(void *arg) ...@@ -654,8 +678,8 @@ pmd_lcore_ldpc_dec(void *arg)
struct thread_params *tp = arg; struct thread_params *tp = arg;
uint16_t enq, deq; uint16_t enq, deq;
const uint16_t queue_id = tp->queue_id; const uint16_t queue_id = tp->queue_id;
const uint16_t burst_sz = tp->op_params->burst_sz; const uint16_t burst_sz = tp->p_offloadParams->C; //tp->op_params->burst_sz;
const uint16_t num_ops = tp->op_params->num_to_process; const uint16_t num_ops = tp->p_offloadParams->C;
struct rte_bbdev_dec_op *ops_enq[num_ops]; struct rte_bbdev_dec_op *ops_enq[num_ops];
struct rte_bbdev_dec_op *ops_deq[num_ops]; struct rte_bbdev_dec_op *ops_deq[num_ops];
struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
...@@ -675,25 +699,23 @@ pmd_lcore_ldpc_dec(void *arg) ...@@ -675,25 +699,23 @@ pmd_lcore_ldpc_dec(void *arg)
bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
rte_pause(); rte_pause();
ret = rte_mempool_get_bulk(tp->op_params->mp_dec, (void **)ops_enq, num_ops);
// looks like a bbdev internal error for the free operation, workaround here
ops_enq[0]->mempool = tp->op_params->mp_dec;
// ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
set_ldpc_dec_op(ops_enq,
num_ops,
0,
bufs,
ref_op,
r,
harq_pid,
ulsch_id,
p_offloadParams);
ret = rte_mempool_get_bulk(tp->op_params->mp_dec, (void **)ops_enq, num_ops);
/*
for (j = 0; j < num_ops; ++j) {
ops_enq[j]->mempool = tp->op_params->mp_dec;
}
*/
set_ldpc_dec_op(ops_enq, num_ops, 0, bufs, ref_op, r, harq_pid, ulsch_id, p_offloadParams);
/* Set counter to validate the ordering */ /* Set counter to validate the ordering */
/*
for (j = 0; j < num_ops; ++j) for (j = 0; j < num_ops; ++j)
ops_enq[j]->opaque_data = (void *)(uintptr_t)j; ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
*/
for (j = 0; j < num_ops; ++j) {
mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data);
}
for (enq = 0, deq = 0; enq < num_ops;) { for (enq = 0, deq = 0; enq < num_ops;) {
num_to_enq = burst_sz; num_to_enq = burst_sz;
...@@ -711,21 +733,28 @@ pmd_lcore_ldpc_dec(void *arg) ...@@ -711,21 +733,28 @@ pmd_lcore_ldpc_dec(void *arg)
time_out++; time_out++;
DevAssert(time_out <= TIME_OUT_POLL); DevAssert(time_out <= TIME_OUT_POLL);
} }
// This if statement is not in DPDK // This if statement is not in DPDK
if (deq == enq) { if (deq == enq) {
tp->iter_count = 0; tp->iter_count = 0;
/* get the max of iter_count for all dequeued ops */ /* get the max of iter_count for all dequeued ops */
for (i = 0; i < num_ops; ++i) { for (i = 0; i < num_ops; ++i) {
uint8_t *status = tp->p_offloadParams->status_cb[i];
tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, tp->iter_count); tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, tp->iter_count);
*status = ops_enq[i]->status;
} }
ret = retrieve_ldpc_dec_op(ops_deq, num_ops, ref_op, tp->op_params->vector_mask, p_out); ret = retrieve_ldpc_dec_op(ops_deq, num_ops, ref_op, tp->op_params->vector_mask, p_out);
TEST_ASSERT_SUCCESS(ret, "Validation failed!"); TEST_ASSERT_SUCCESS(ret, "Validation failed!");
} }
/*
for (int k = 0; k < num_ops; k++) {
rte_mempool_put_bulk(ops_enq[k]->mempool, (void **)ops_enq, num_ops);
}
*/
for (j = 0; j < num_ops; ++j) {
mbuf_reset(ops_enq[j]->ldpc_dec.input.data);
}
if (num_ops > 0) rte_bbdev_dec_op_free_bulk(ops_enq, num_ops);
rte_mempool_put_bulk(ops_enq[0]->mempool, (void **)ops_enq, num_ops);
// Return the worst decoding number of iterations for all segments // Return the worst decoding number of iterations for all segments
return tp->iter_count; return tp->iter_count;
} }
...@@ -736,12 +765,14 @@ static int pmd_lcore_ldpc_enc(void *arg) ...@@ -736,12 +765,14 @@ static int pmd_lcore_ldpc_enc(void *arg)
struct thread_params *tp = arg; struct thread_params *tp = arg;
uint16_t enq, deq; uint16_t enq, deq;
const uint16_t queue_id = tp->queue_id; const uint16_t queue_id = tp->queue_id;
const uint16_t burst_sz = tp->op_params->burst_sz; const uint16_t burst_sz = tp->p_offloadParams->C;
tp->op_params->num_to_process = tp->p_offloadParams->C;
const uint16_t num_ops = tp->op_params->num_to_process; const uint16_t num_ops = tp->op_params->num_to_process;
struct rte_bbdev_enc_op *ops_enq[num_ops]; struct rte_bbdev_enc_op *ops_enq[num_ops];
struct rte_bbdev_enc_op *ops_deq[num_ops]; struct rte_bbdev_enc_op *ops_deq[num_ops];
struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
int j, ret; int j, ret;
struct test_buffers *bufs = NULL;
uint16_t num_to_enq; uint16_t num_to_enq;
uint8_t *p_out = tp->p_out; uint8_t *p_out = tp->p_out;
t_nrLDPCoffload_params *p_offloadParams = tp->p_offloadParams; t_nrLDPCoffload_params *p_offloadParams = tp->p_offloadParams;
...@@ -753,14 +784,12 @@ static int pmd_lcore_ldpc_enc(void *arg) ...@@ -753,14 +784,12 @@ static int pmd_lcore_ldpc_enc(void *arg)
TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), "NUM_OPS cannot exceed %u for this device", info.drv.queue_size_lim); TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), "NUM_OPS cannot exceed %u for this device", info.drv.queue_size_lim);
struct test_buffers *bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
rte_pause(); rte_pause();
ret = rte_mempool_get_bulk(tp->op_params->mp_enc, (void **)ops_enq, num_ops); ret = rte_mempool_get_bulk(tp->op_params->mp_enc, (void **)ops_enq, num_ops);
// ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
ops_enq[0]->mempool = tp->op_params->mp_enc; ops_enq[0]->mempool = tp->op_params->mp_enc;
set_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, bufs->hard_outputs, ref_op, p_offloadParams); set_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, bufs->hard_outputs, ref_op, p_offloadParams);
...@@ -788,13 +817,12 @@ static int pmd_lcore_ldpc_enc(void *arg) ...@@ -788,13 +817,12 @@ static int pmd_lcore_ldpc_enc(void *arg)
DevAssert(time_out <= TIME_OUT_POLL); DevAssert(time_out <= TIME_OUT_POLL);
} }
ret = retrieve_ldpc_enc_op(ops_deq, num_ops, ref_op, p_out); ret = retrieve_ldpc_enc_op(ops_deq, num_ops, ref_op, p_out, tp->p_offloadParams->E_cb);
TEST_ASSERT_SUCCESS(ret, "Validation failed!"); TEST_ASSERT_SUCCESS(ret, "Validation failed!");
// rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); //rte_bbdev_enc_op_free_bulk(ops_enq, num_ops);
if (num_ops > 0) if (num_ops > 0)
rte_mempool_put_bulk(ops_enq[0]->mempool, (void **)ops_enq, num_ops); rte_mempool_put_bulk(ops_enq[0]->mempool, (void **)ops_enq, num_ops);
return ret; return ret;
} }
...@@ -841,7 +869,6 @@ int start_pmd_dec(struct active_device *ad, ...@@ -841,7 +869,6 @@ int start_pmd_dec(struct active_device *ad,
t_params[0].r = r; t_params[0].r = r;
t_params[0].harq_pid = harq_pid; t_params[0].harq_pid = harq_pid;
t_params[0].ulsch_id = ulsch_id; t_params[0].ulsch_id = ulsch_id;
// For now, we never enter here, we don't use the DPDK thread pool // For now, we never enter here, we don't use the DPDK thread pool
RTE_LCORE_FOREACH_WORKER(lcore_id) { RTE_LCORE_FOREACH_WORKER(lcore_id) {
if (used_cores >= num_lcores) if (used_cores >= num_lcores)
...@@ -867,12 +894,6 @@ int start_pmd_dec(struct active_device *ad, ...@@ -867,12 +894,6 @@ int start_pmd_dec(struct active_device *ad,
// for (used_cores = 1; used_cores < num_lcores; used_cores++) // for (used_cores = 1; used_cores < num_lcores; used_cores++)
// ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); // ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
/* Return if test failed */
if (ret < 0) {
rte_free(t_params);
return ret;
}
rte_free(t_params); rte_free(t_params);
return ret; return ret;
} }
...@@ -944,7 +965,7 @@ int32_t LDPCinit() ...@@ -944,7 +965,7 @@ int32_t LDPCinit()
char *dpdk_dev = "41:00.0"; //PCI address of the card char *dpdk_dev = "41:00.0"; //PCI address of the card
char *argv_re[] = {"bbdev", "-a", dpdk_dev, "-l", "14-15", "--file-prefix=b6", "--"}; char *argv_re[] = {"bbdev", "-a", dpdk_dev, "-l", "14-15", "--file-prefix=b6", "--"};
// EAL initialization, if already initialized (init in xran lib) try to probe DPDK device // EAL initialization, if already initialized (init in xran lib) try to probe DPDK device
ret = rte_eal_init(5, argv_re); ret = rte_eal_init(7, argv_re);
if (ret < 0) { if (ret < 0) {
printf("EAL initialization failed, probing DPDK device %s\n", dpdk_dev); printf("EAL initialization failed, probing DPDK device %s\n", dpdk_dev);
if (rte_dev_probe(dpdk_dev) != 0) { if (rte_dev_probe(dpdk_dev) != 0) {
...@@ -969,8 +990,8 @@ int32_t LDPCinit() ...@@ -969,8 +990,8 @@ int32_t LDPCinit()
int socket_id = GET_SOCKET(info.socket_id); int socket_id = GET_SOCKET(info.socket_id);
int out_max_sz = 8448; // max code block size (for BG1), 22 * 384 int out_max_sz = 8448; // max code block size (for BG1), 22 * 384
int in_max_sz = LDPC_MAX_CB_SIZE; // max number of encoded bits (for BG2 and MCS0) int in_max_sz = LDPC_MAX_CB_SIZE; // max number of encoded bits (for BG2 and MCS0)
int num_ops = 1; int num_queues = 1;
int f_ret = create_mempools(ad, socket_id, num_ops, out_max_sz, in_max_sz); int f_ret = create_mempools(ad, socket_id, num_queues, out_max_sz, in_max_sz);
if (f_ret != TEST_SUCCESS) { if (f_ret != TEST_SUCCESS) {
printf("Couldn't create mempools"); printf("Couldn't create mempools");
return -1; return -1;
...@@ -978,41 +999,12 @@ int32_t LDPCinit() ...@@ -978,41 +999,12 @@ int32_t LDPCinit()
// get_num_lcores() hardcoded to 1: we use one core for decode, and another for encode // get_num_lcores() hardcoded to 1: we use one core for decode, and another for encode
// this code from bbdev test example is not considering encode and decode test // this code from bbdev test example is not considering encode and decode test
// get_num_ops() replaced by 1: LDPC decode and ldpc encode (7th param) // get_num_ops() replaced by 1: LDPC decode and ldpc encode (7th param)
f_ret = init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_DEC, ad->bbdev_dec_op_pool, 1, 1, 1); f_ret = init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_DEC, ad->bbdev_dec_op_pool, num_queues, num_queues, 1);
f_ret |= init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_ENC, ad->bbdev_enc_op_pool, 1, 1, 1); f_ret = init_test_op_params(op_params, RTE_BBDEV_OP_LDPC_ENC, ad->bbdev_enc_op_pool, num_queues, num_queues, 1);
if (f_ret != TEST_SUCCESS) { if (f_ret != TEST_SUCCESS) {
printf("Couldn't init test op params"); printf("Couldn't init test op params");
return -1; return -1;
} }
// fill_queue_buffers -> allocate_buffers_on_socket
for (int i = 0; i < ad->nb_queues; ++i) {
const uint16_t n = op_params->num_to_process;
struct rte_mempool *in_mp = ad->in_mbuf_pool;
struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool;
struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool;
struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool;
struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool;
struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {in_mp, soft_out_mp, hard_out_mp, harq_in_mp, harq_out_mp};
uint8_t queue_id = ad->queue_ids[i];
struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&op_params->q_bufs[socket_id][queue_id].inputs,
&op_params->q_bufs[socket_id][queue_id].soft_outputs,
&op_params->q_bufs[socket_id][queue_id].hard_outputs,
&op_params->q_bufs[socket_id][queue_id].harq_inputs,
&op_params->q_bufs[socket_id][queue_id].harq_outputs};
for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) {
int ret = allocate_buffers_on_socket(queue_ops[type], n * sizeof(struct rte_bbdev_op_data), socket_id);
TEST_ASSERT_SUCCESS(ret, "Couldn't allocate memory for rte_bbdev_op_data structs");
m_head[type] = rte_pktmbuf_alloc(mbuf_pools[type]);
TEST_ASSERT_NOT_NULL(m_head[type],
"Not enough mbufs in %d data type mbuf pool (needed %d, available %u)",
type,
1,
mbuf_pools[type]->size);
}
}
return 0; return 0;
} }
...@@ -1043,7 +1035,7 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams, ...@@ -1043,7 +1035,7 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams,
{ {
pthread_mutex_lock(&decode_mutex); pthread_mutex_lock(&decode_mutex);
// hardcoded we use first device // hardcoded we use first device
LOG_D(PHY,"Offload\n");
struct active_device *ad = active_devs; struct active_device *ad = active_devs;
t_nrLDPCoffload_params offloadParams = {.E = p_decParams->E, t_nrLDPCoffload_params offloadParams = {.E = p_decParams->E,
.n_cb = (p_decParams->BG == 1) ? (66 * p_decParams->Z) : (50 * p_decParams->Z), .n_cb = (p_decParams->BG == 1) ? (66 * p_decParams->Z) : (50 * p_decParams->Z),
...@@ -1052,8 +1044,16 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams, ...@@ -1052,8 +1044,16 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams,
.rv = p_decParams->rv, .rv = p_decParams->rv,
.F = p_decParams->F, .F = p_decParams->F,
.Qm = p_decParams->Qm, .Qm = p_decParams->Qm,
.numMaxIter = p_decParams->numMaxIter,
.C = C,
.setCombIn = p_decParams->setCombIn}; .setCombIn = p_decParams->setCombIn};
for (int r = 0; r < C; r++) {
offloadParams.E_cb[r] = p_decParams->E_cb[r];
offloadParams.status_cb[r] = &p_decParams->status_cb[r];
}
struct rte_bbdev_info info; struct rte_bbdev_info info;
int ret;
rte_bbdev_info_get(ad->dev_id, &info); rte_bbdev_info_get(ad->dev_id, &info);
int socket_id = GET_SOCKET(info.socket_id); int socket_id = GET_SOCKET(info.socket_id);
// fill_queue_buffers -> init_op_data_objs // fill_queue_buffers -> init_op_data_objs
...@@ -1072,21 +1072,23 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams, ...@@ -1072,21 +1072,23 @@ int32_t LDPCdecoder(struct nrLDPC_dec_params *p_decParams,
// this should be modified // this should be modified
// enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC; // enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) { for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) {
int ret = init_op_data_objs(*queue_ops[type], ret = allocate_buffers_on_socket(queue_ops[type], C * sizeof(struct rte_bbdev_op_data), socket_id);
TEST_ASSERT_SUCCESS(ret, "Couldn't allocate memory for rte_bbdev_op_data structs");
//rte_pktmbuf_free(m_head[type]);
ret = init_op_data_objs_dec(*queue_ops[type],
(uint8_t *)p_llr, (uint8_t *)p_llr,
p_decParams->E, &offloadParams,
m_head[type],
mbuf_pools[type], mbuf_pools[type],
1, C,
type, type,
info.drv.min_alignment); info.drv.min_alignment);
TEST_ASSERT_SUCCESS(ret, "Couldn't init rte_bbdev_op_data structs"); TEST_ASSERT_SUCCESS(ret, "Couldn't init rte_bbdev_op_data structs");
} }
int ret = start_pmd_dec(ad, op_params, &offloadParams, C, harq_pid, ulsch_id, (uint8_t *)p_out); ret = start_pmd_dec(ad, op_params, &offloadParams, C, harq_pid, ulsch_id, (uint8_t *)p_out);
if (ret < 0) { if (ret < 0) {
printf("Couldn't start pmd dec\n"); printf("Couldn't start pmd dec\n");
pthread_mutex_unlock(&decode_mutex); pthread_mutex_unlock(&decode_mutex);
return (20); // Fix me: we should propoagate max_iterations properly in the call (impp struct) return (p_decParams->numMaxIter); // Fix me: we should propoagate max_iterations properly in the call (impp struct)
} }
pthread_mutex_unlock(&decode_mutex); pthread_mutex_unlock(&decode_mutex);
return ret; return ret;
...@@ -1097,16 +1099,20 @@ int32_t LDPCencoder(unsigned char **input, unsigned char **output, encoder_imple ...@@ -1097,16 +1099,20 @@ int32_t LDPCencoder(unsigned char **input, unsigned char **output, encoder_imple
pthread_mutex_lock(&encode_mutex); pthread_mutex_lock(&encode_mutex);
// hardcoded to use the first found board // hardcoded to use the first found board
struct active_device *ad = active_devs; struct active_device *ad = active_devs;
int Zc = impp->Zc; int ret;
int BG = impp->BG;
t_nrLDPCoffload_params offloadParams = {.E = impp->E, t_nrLDPCoffload_params offloadParams = {.E = impp->E,
.n_cb = (BG == 1) ? (66 * Zc) : (50 * Zc), .n_cb = (impp->BG == 1) ? (66 * impp->Zc) : (50 * impp->Zc),
.BG = BG, .BG = impp->BG,
.Z = Zc, .Z = impp->Zc,
.rv = impp->rv, .rv = impp->rv,
.F = impp->F, .F = impp->F,
.Qm = impp->Qm, .Qm = impp->Qm,
.C = impp->n_segments,
.Kr = (impp->K - impp->F + 7)/8}; .Kr = (impp->K - impp->F + 7)/8};
for (int r = 0; r < impp->n_segments; r++) {
offloadParams.E_cb[r] = impp->E_cb[r];
}
struct rte_bbdev_info info; struct rte_bbdev_info info;
rte_bbdev_info_get(ad->dev_id, &info); rte_bbdev_info_get(ad->dev_id, &info);
int socket_id = GET_SOCKET(info.socket_id); int socket_id = GET_SOCKET(info.socket_id);
...@@ -1124,17 +1130,20 @@ int32_t LDPCencoder(unsigned char **input, unsigned char **output, encoder_imple ...@@ -1124,17 +1130,20 @@ int32_t LDPCencoder(unsigned char **input, unsigned char **output, encoder_imple
&op_params->q_bufs[socket_id][queue_id].harq_inputs, &op_params->q_bufs[socket_id][queue_id].harq_inputs,
&op_params->q_bufs[socket_id][queue_id].harq_outputs}; &op_params->q_bufs[socket_id][queue_id].harq_outputs};
for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) { for (enum op_data_type type = DATA_INPUT; type < 3; type += 2) {
int ret = init_op_data_objs(*queue_ops[type], ret = allocate_buffers_on_socket(queue_ops[type], impp->n_segments * sizeof(struct rte_bbdev_op_data), socket_id);
*input, TEST_ASSERT_SUCCESS(ret, "Couldn't allocate memory for rte_bbdev_op_data structs");
offloadParams.Kr, //m_head[type] = rte_pktmbuf_alloc(mbuf_pools[type]);
ret = init_op_data_objs_enc(*queue_ops[type],
input,
&offloadParams,
m_head[type], m_head[type],
mbuf_pools[type], mbuf_pools[type],
1, impp->n_segments,
type, type,
info.drv.min_alignment); info.drv.min_alignment);
TEST_ASSERT_SUCCESS(ret, "Couldn't init rte_bbdev_op_data structs"); TEST_ASSERT_SUCCESS(ret, "Couldn't init rte_bbdev_op_data structs");
} }
int ret=start_pmd_enc(ad, op_params, &offloadParams, *output); ret = start_pmd_enc(ad, op_params, &offloadParams, *output);
pthread_mutex_unlock(&encode_mutex); pthread_mutex_unlock(&encode_mutex);
return ret; return ret;
} }
...@@ -90,10 +90,13 @@ typedef struct nrLDPC_dec_params { ...@@ -90,10 +90,13 @@ typedef struct nrLDPC_dec_params {
uint8_t rv; uint8_t rv;
uint8_t numMaxIter; /**< Maximum number of iterations */ uint8_t numMaxIter; /**< Maximum number of iterations */
int E; int E;
uint32_t E_cb[64];
e_nrLDPC_outMode outMode; /**< Output format */ e_nrLDPC_outMode outMode; /**< Output format */
int crc_type; int crc_type;
int (*check_crc)(uint8_t* decoded_bytes, uint32_t n, uint8_t crc_type); int (*check_crc)(uint8_t* decoded_bytes, uint32_t n, uint8_t crc_type);
uint8_t setCombIn; uint8_t setCombIn;
uint8_t setCombIn_cb[64];
uint8_t status_cb[64];
} t_nrLDPC_dec_params; } t_nrLDPC_dec_params;
typedef struct nrLDPCoffload_params { typedef struct nrLDPCoffload_params {
...@@ -102,10 +105,15 @@ typedef struct nrLDPCoffload_params { ...@@ -102,10 +105,15 @@ typedef struct nrLDPCoffload_params {
uint16_t Kr; uint16_t Kr;
uint8_t rv; uint8_t rv;
uint32_t E; uint32_t E;
uint32_t E_cb[64];
uint16_t n_cb; uint16_t n_cb;
uint16_t F; /**< Filler bits */ uint16_t F; /**< Filler bits */
uint8_t Qm; /**< Modulation */ uint8_t Qm; /**< Modulation */
uint8_t C;
uint8_t numMaxIter;
uint8_t setCombIn; uint8_t setCombIn;
uint8_t setCombIn_cb[64];
uint8_t *status_cb[64];
} t_nrLDPCoffload_params; } t_nrLDPCoffload_params;
/** /**
......
...@@ -60,6 +60,8 @@ typedef struct { ...@@ -60,6 +60,8 @@ typedef struct {
/// Modulation order /// Modulation order
uint8_t Qm; uint8_t Qm;
uint32_t E; uint32_t E;
uint32_t E_cb[64];
uint8_t status_cb[64];
unsigned int G; unsigned int G;
// Redundancy version index // Redundancy version index
uint8_t rv; uint8_t rv;
......
...@@ -366,17 +366,14 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB, ...@@ -366,17 +366,14 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
if (gNB->ldpc_offload_flag) { if (gNB->ldpc_offload_flag) {
impp.Qm = rel15->qamModOrder[0]; impp.Qm = rel15->qamModOrder[0];
impp.rv = rel15->rvIndex[0]; impp.rv = rel15->rvIndex[0];
int nb_re_dmrs = int nb_re_dmrs = (rel15->dmrsConfigType == NFAPI_NR_DMRS_TYPE1) ?
(rel15->dmrsConfigType == NFAPI_NR_DMRS_TYPE1) ? (6 * rel15->numDmrsCdmGrpsNoData) : (4 * rel15->numDmrsCdmGrpsNoData); (6 * rel15->numDmrsCdmGrpsNoData) : (4 * rel15->numDmrsCdmGrpsNoData);
impp.G = nr_get_G(rel15->rbSize, rel15->NrOfSymbols, nb_re_dmrs, get_num_dmrs(rel15->dlDmrsSymbPos), harq->unav_res, impp.G = nr_get_G(rel15->rbSize, rel15->NrOfSymbols, nb_re_dmrs, get_num_dmrs(rel15->dlDmrsSymbPos),
rel15->qamModOrder[0], rel15->nrOfLayers); harq->unav_res, rel15->qamModOrder[0], rel15->nrOfLayers);
int r_offset = 0;
for (int r = 0; r < impp.n_segments; r++) { for (int r = 0; r < impp.n_segments; r++) {
impp.E = nr_get_E(impp.G, impp.n_segments, impp.Qm, rel15->nrOfLayers, r); impp.E_cb[r] = nr_get_E(impp.G, impp.n_segments, impp.Qm, rel15->nrOfLayers, r);
uint8_t *f = impp.output + r_offset;
ldpc_interface_offload.LDPCencoder(&harq->c[r], &f, &impp);
r_offset += impp.E;
} }
ldpc_interface_offload.LDPCencoder(harq->c, &impp.output, &impp);
} else { } else {
notifiedFIFO_t nf; notifiedFIFO_t nf;
initNotifiedFIFO(&nf); initNotifiedFIFO(&nf);
......
...@@ -232,59 +232,50 @@ int decode_offload(PHY_VARS_gNB *phy_vars_gNB, ...@@ -232,59 +232,50 @@ int decode_offload(PHY_VARS_gNB *phy_vars_gNB,
{ {
NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id]; NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id];
NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process; NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
int16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); int16_t z_ol[64 * LDPC_MAX_CB_SIZE] __attribute__((aligned(32)));
int8_t l_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16))); int8_t l_ol[64 * LDPC_MAX_CB_SIZE] __attribute__((aligned(32)));
uint8_t Qm = pusch_pdu->qam_mod_order; const int kc = decParams->BG == 2 ? 52 : 68;
uint8_t n_layers = pusch_pdu->nrOfLayers; uint32_t A = (harq_process->TBS) << 3;
const int Kr = harq_process->K; const int Kr = harq_process->K;
const int Kr_bytes = Kr >> 3; const int Kr_bytes = Kr >> 3;
uint32_t A = (harq_process->TBS) << 3; int8_t decodeIterations = 0;
const int kc = decParams->BG == 2 ? 52 : 68; uint8_t *p_outDec;
ulsch->max_ldpc_iterations = 20; int r_offset = 0;
int decodeIterations = 2; int offset = 0;
int r_offset = 0, offset = 0;
p_outDec = calloc(64 * Kr_bytes, sizeof(int8_t));
for (int r = 0; r < harq_process->C; r++) { for (int r = 0; r < harq_process->C; r++) {
int E = nr_get_E(G, harq_process->C, Qm, n_layers, r); decParams->E_cb[r] = nr_get_E(G, harq_process->C, decParams->Qm, pusch_pdu->nrOfLayers, r);
memset(harq_process->c[r], 0, Kr_bytes); memcpy(&z_ol[offset], ulsch_llr + r_offset, decParams->E_cb[r] * sizeof(short));
decParams->R = nr_get_R_ldpc_decoder(pusch_pdu->pusch_data.rv_index, simde__m128i *pv_ol128 = (simde__m128i *)&z_ol[offset];
E, simde__m128i *pl_ol128 = (simde__m128i *)&l_ol[offset];
decParams->BG,
decParams->Z,
&harq_process->llrLen,
harq_process->round);
memcpy(z_ol, ulsch_llr + r_offset, E * sizeof(short));
simde__m128i *pv_ol128 = (simde__m128i *)&z_ol;
simde__m128i *pl_ol128 = (simde__m128i *)&l_ol;
for (int i = 0, j = 0; j < ((kc * harq_process->Z) >> 4) + 1; i += 2, j++) { for (int i = 0, j = 0; j < ((kc * harq_process->Z) >> 4) + 1; i += 2, j++) {
pl_ol128[j] = simde_mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]); pl_ol128[j] = simde_mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]);
} }
decParams->E = E;
decParams->rv = pusch_pdu->pusch_data.rv_index;
decParams->F = harq_process->F; decParams->F = harq_process->F;
decParams->Qm = Qm; r_offset += decParams->E_cb[r];
decodeIterations = offset += LDPC_MAX_CB_SIZE;
ldpc_interface_offload }
.LDPCdecoder(decParams, harq_pid, ULSCH_id, r, (int8_t *)&pl_ol128[0], (int8_t *)harq_process->c[r], NULL, NULL); decodeIterations = ldpc_interface_offload.LDPCdecoder(decParams, harq_pid, ULSCH_id, harq_process->C, (int8_t *)l_ol, (int8_t *)p_outDec, NULL, NULL);
if (decodeIterations < 0) {
LOG_E(PHY, "ulsch_decoding.c: Problem in LDPC decoder offload\n"); if (decodeIterations < 0) {
return -1; LOG_E(PHY, "ulsch_decoding.c: Problem in LDPC decoder offload\n");
} return -1;
bool decodeSuccess = check_crc((uint8_t *)harq_process->c[r], lenWithCrc(harq_process->C, A), crcType(harq_process->C, A)); }
if (decodeSuccess) {
memcpy(harq_process->b + offset, harq_process->c[r], Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0)); int offset_b = 0;
offset += (Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0)); for (int r = 0; r < harq_process->C; r++) {
harq_process->processedSegments++; if (decParams->status_cb[r] == 0) {
} else { memcpy(harq_process->b + offset_b, &p_outDec[offset_b], Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
LOG_D(PHY, "uplink segment error %d/%d\n", r, harq_process->C);
LOG_D(PHY, "ULSCH %d in error\n", ULSCH_id);
} }
r_offset += E; offset_b += (Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
harq_process->processedSegments++;
} }
bool crc_valid = false; bool crc_valid = false;
if (harq_process->processedSegments == harq_process->C) { //When the number of code blocks is 1 (C = 1) and ulsch_harq->processedSegments = 1, we can assume a good TB because of the
// When the number of code blocks is 1 (C = 1) and ulsch_harq->processedSegments = 1, we can assume a good TB because of the //CRC check made by the LDPC for early termination, so, no need to perform CRC check twice for a single code block
// CRC check made by the LDPC for early termination, so, no need to perform CRC check twice for a single code block if (decodeIterations < ulsch->max_ldpc_iterations) {
crc_valid = true; crc_valid = true;
if (harq_process->C > 1) { if (harq_process->C > 1) {
crc_valid = check_crc(harq_process->b, lenWithCrc(1, A), crcType(1, A)); crc_valid = check_crc(harq_process->b, lenWithCrc(1, A), crcType(1, A));
...@@ -419,6 +410,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -419,6 +410,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
decParams.Z = harq_process->Z; decParams.Z = harq_process->Z;
decParams.numMaxIter = ulsch->max_ldpc_iterations; decParams.numMaxIter = ulsch->max_ldpc_iterations;
decParams.Qm = Qm;
decParams.rv = pusch_pdu->pusch_data.rv_index;
decParams.outMode = 0; decParams.outMode = 0;
decParams.setCombIn = !harq_process->harq_to_be_cleared; decParams.setCombIn = !harq_process->harq_to_be_cleared;
if (harq_process->harq_to_be_cleared) { if (harq_process->harq_to_be_cleared) {
......
...@@ -154,7 +154,7 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, ...@@ -154,7 +154,7 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue,
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_LDPC_ENCODER_OPTIM, VCD_FUNCTION_IN); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_LDPC_ENCODER_OPTIM, VCD_FUNCTION_IN);
} }
start_meas(&ue->ulsch_ldpc_encoding_stats); start_meas(&ue->ulsch_ldpc_encoding_stats);
if (ldpc_interface_offload.LDPCencoder) { if (0) {
for (int j = 0; j < impp.n_segments; j++) { for (int j = 0; j < impp.n_segments; j++) {
impp.E = nr_get_E(G, impp.n_segments, impp.Qm, ulsch->pusch_pdu.nrOfLayers, j); impp.E = nr_get_E(G, impp.n_segments, impp.Qm, ulsch->pusch_pdu.nrOfLayers, j);
uint8_t *f = harq_process->f + r_offset; uint8_t *f = harq_process->f + r_offset;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment